Replication of Goldman et al.

library(ggplot2)
library(tidyverse)
── Attaching core tidyverse packages ───────────────────────────────────────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.1.4     ✔ readr     2.1.5
✔ forcats   1.0.0     ✔ stringr   1.5.1
✔ lubridate 1.9.4     ✔ tibble    3.2.1
✔ purrr     1.0.2     ✔ tidyr     1.3.1── Conflicts ─────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the ]8;;http://conflicted.r-lib.org/conflicted package]8;; to force all conflicts to become errors
library(car)
Loading required package: carData

Attaching package: ‘car’

The following object is masked from ‘package:dplyr’:

    recode

The following object is masked from ‘package:purrr’:

    some
library(mgcv)
Loading required package: nlme

Attaching package: ‘nlme’

The following object is masked from ‘package:dplyr’:

    collapse

This is mgcv 1.9-1. For overview type 'help("mgcv-package")'.
require(gam)
Loading required package: gam
Loading required package: splines
Loading required package: foreach

Attaching package: ‘foreach’

The following objects are masked from ‘package:purrr’:

    accumulate, when

Loaded gam 1.22-5


Attaching package: ‘gam’

The following objects are masked from ‘package:mgcv’:

    gam, gam.control, gam.fit, s
require(lmtest)
Loading required package: lmtest
Loading required package: zoo

Attaching package: ‘zoo’

The following objects are masked from ‘package:base’:

    as.Date, as.Date.numeric
require(lme4)
Loading required package: lme4
Loading required package: Matrix

Attaching package: ‘Matrix’

The following objects are masked from ‘package:tidyr’:

    expand, pack, unpack


Attaching package: ‘lme4’

The following object is masked from ‘package:nlme’:

    lmList

Load in & Preprocess the Data

df<-read.csv("replication_final.csv")
df$Family = str_to_title(str_replace(df$Family, "_", " "))

CHRTRM <- df %>% 
  select(c(Language,
           Family,
           SIGMORPHON_acc = CHR.TRM_SIGMORPHON_test_acc, 
           Goldman_acc = CHR.TRM_Goldman_test_acc, 
           Goldman_train_size, 
           Goldman_train_lemmas,
           train_lemma_diff_raw, 
           SIGMORPHON_train_size, 
           SIGMORPHON_train_lemmas
           ))

CHRTRM$Model = "CHR-TRM"



CLUZH <- df %>% 
  select(c(Language,
           Family,
           SIGMORPHON_acc = CLUZH_SIGMORPHON_test_acc, 
           Goldman_acc = CLUZH_Goldman_test_acc, 
           Goldman_train_size, 
           Goldman_train_lemmas,
           train_lemma_diff_raw,
           SIGMORPHON_train_size,
           SIGMORPHON_train_lemmas
           ))


CLUZH$Model = "CLUZH"



DeepSpin <- df %>% 
  select(c(Language,
           Family,
           SIGMORPHON_acc = DeepSpin_SIGMORPHON_test_acc, 
           Goldman_acc = DeepSpin_Goldman_test_acc, 
           Goldman_train_size, 
           Goldman_train_lemmas,
           train_lemma_diff_raw,
           SIGMORPHON_train_size,
           SIGMORPHON_train_lemmas
           ))

DeepSpin$Model = "DeepSpin"



all_data <- rbind(CLUZH, CHRTRM, DeepSpin)



all_data$acc_drop = all_data$Goldman_acc - all_data$SIGMORPHON_acc
all_data$log_acc_drop <- -1*log(-1*(all_data$acc_drop - 4))
max(all_data$acc_drop)
[1] 2.0202
# We'll use this helper function to run our stats
correlations <- function(a, b){
  for (m in list("pearson", "spearman", "kendall")){
    # Supressing warnings bc we'll get them whenever there are ties 
    suppressWarnings(res <- cor.test(a, b, method = m))
    formatted <- sprintf("%s: %f (p = %f)", res$method, res$estimate, res$p.value)
    print(formatted)
  }
}

# We'll use this helper function to get information about our model
eval_model <- function(model, df){
  rsquared = summary(model)$r.squared
  AIC = AIC(model)
  results <- sprintf("R^2: %f, AIC: %f", rsquared, AIC)
  print(results)
  layout(matrix(c(1,2,3,4),2,2)) 
  plot(model)
  return(predict(model, df, se = TRUE))
}

# We'll use this helper function to compare two models 
compare_models <- function(model_both, model_single){
  layout(matrix(c(1,2,3,4),2,2)) 
  plot(model_both)
  AIC = AIC(model_both)
  anovap = anova(model_both, model_single)$`Pr(>F)`[-1]
  results <- sprintf("AIC: %f, ANOVA p: %f", AIC, anovap)
  print(results)
  vif(model_both)
}

Raw Accuracy

Initial Scatters


SIGMORPHON <- all_data %>% 
  select(c(Model, Family, train_size=SIGMORPHON_train_size, train_lemmas=SIGMORPHON_train_lemmas, test_acc = SIGMORPHON_acc))
SIGMORPHON$Data <- "SIGMORPHON" 

Goldman <- all_data %>%
  select(c(Model, Family, train_size=Goldman_train_size, train_lemmas=Goldman_train_lemmas, test_acc = Goldman_acc))
Goldman$Data <- "Goldman et al."

rbind(SIGMORPHON, Goldman) %>%
  ggplot(aes(log(train_size), log(test_acc), color=Data)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH")))+
  scale_color_manual(values=c("turquoise", "purple")) +
  geom_point(aes(shape=Family), size=3, alpha=0.6) + 
  theme_bw() + 
  xlab("Training Size, Log Scale") + 
  ylab("Test Accuracy, Log Scale") + 
  ggtitle("Test Accuracy vs. Training Size") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/train_size_accuracy.pdf", device="pdf", width = 12, height=4)
rbind(SIGMORPHON, Goldman) %>%
  ggplot(aes(log(train_lemmas), log(test_acc), color=Data)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH")))+
  scale_color_manual(values=c("turquoise", "purple")) +
  geom_point(aes(shape=Family), size=3, alpha=0.6) + 
  theme_bw() + 
  xlab("Training Lemmas, Log Scale") + 
  ylab("Test Accuracy, Log Scale") + 
  ggtitle("Test Accuracy vs. Training Lemmas") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/train_lemmas_accuracy.pdf", device="pdf", width = 12, height=4)

Basic Correlational Stats

correlations(log(CHRTRM$Goldman_train_size), log(CHRTRM$Goldman_acc + 1))
[1] "Pearson's product-moment correlation: 0.703670 (p = 0.000003)"
[1] "Spearman's rank correlation rho: 0.817515 (p = 0.000000)"
[1] "Kendall's rank correlation tau: 0.621986 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(DeepSpin$Goldman_train_size), log(DeepSpin$Goldman_acc + 1))
[1] "Pearson's product-moment correlation: 0.411709 (p = 0.015553)"
[1] "Spearman's rank correlation rho: 0.028615 (p = 0.872373)"
[1] "Kendall's rank correlation tau: -0.012590 (p = 0.917200)"
sprintf("")
[1] ""
correlations(log(CLUZH$Goldman_train_size), log(CLUZH$Goldman_acc + 1))
[1] "Pearson's product-moment correlation: 0.509641 (p = 0.002079)"
[1] "Spearman's rank correlation rho: 0.062892 (p = 0.723823)"
[1] "Kendall's rank correlation tau: 0.019785 (p = 0.870228)"
correlations(log(CHRTRM$SIGMORPHON_train_size), log(CHRTRM$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.308926 (p = 0.075448)"
[1] "Spearman's rank correlation rho: 0.039633 (p = 0.823893)"
[1] "Kendall's rank correlation tau: 0.008993 (p = 0.940805)"
sprintf("")
[1] ""
correlations(log(DeepSpin$SIGMORPHON_train_size), log(DeepSpin$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.318779 (p = 0.066132)"
[1] "Spearman's rank correlation rho: -0.028388 (p = 0.873380)"
[1] "Kendall's rank correlation tau: -0.058140 (p = 0.633569)"
sprintf("")
[1] ""
correlations(log(CLUZH$SIGMORPHON_train_size), log(CLUZH$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.401506 (p = 0.018593)"
[1] "Spearman's rank correlation rho: 0.120893 (p = 0.495836)"
[1] "Kendall's rank correlation tau: 0.078642 (p = 0.514054)"
correlations(log(CHRTRM$Goldman_train_lemmas), log(CHRTRM$Goldman_acc+1))
[1] "Pearson's product-moment correlation: 0.663836 (p = 0.000019)"
[1] "Spearman's rank correlation rho: 0.849805 (p = 0.000000)"
[1] "Kendall's rank correlation tau: 0.646953 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(DeepSpin$Goldman_train_lemmas), log(DeepSpin$Goldman_acc+1))
[1] "Pearson's product-moment correlation: 0.239558 (p = 0.172389)"
[1] "Spearman's rank correlation rho: -0.011326 (p = 0.949309)"
[1] "Kendall's rank correlation tau: -0.057711 (p = 0.634496)"
sprintf("")
[1] ""
correlations(log(CLUZH$Goldman_train_lemmas), log(CLUZH$Goldman_acc+1))
[1] "Pearson's product-moment correlation: 0.381134 (p = 0.026152)"
[1] "Spearman's rank correlation rho: 0.061529 (p = 0.729582)"
[1] "Kendall's rank correlation tau: -0.010821 (p = 0.928971)"
correlations(log(CHRTRM$SIGMORPHON_train_lemmas), log(CHRTRM$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.155920 (p = 0.378556)"
[1] "Spearman's rank correlation rho: -0.002142 (p = 0.990405)"
[1] "Kendall's rank correlation tau: -0.057608 (p = 0.634570)"
sprintf("")
[1] ""
correlations(log(DeepSpin$SIGMORPHON_train_lemmas), log(DeepSpin$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.161117 (p = 0.362669)"
[1] "Spearman's rank correlation rho: -0.084248 (p = 0.635701)"
[1] "Kendall's rank correlation tau: -0.118201 (p = 0.332834)"
sprintf("")
[1] ""
correlations(log(CLUZH$SIGMORPHON_train_lemmas), log(CLUZH$SIGMORPHON_acc+1))
[1] "Pearson's product-moment correlation: 0.268885 (p = 0.124126)"
[1] "Spearman's rank correlation rho: 0.095224 (p = 0.592169)"
[1] "Kendall's rank correlation tau: 0.030411 (p = 0.800929)"

GLMer Attempt

data <- rbind(SIGMORPHON, Goldman)
data$scaled_acc <- (data$test_acc/100 * 203 + .5) / 204


model <- glmer(scaled_acc ~ log(train_lemmas) + log(train_size) + (1|Model)  + ( train_size + train_lemmas |Family) + (train_lemmas |Data), data=data, family=binomial)
summary(model)

Beta Regression

rbind(CLUZH, CHRTRM, DeepSpin) %>%
  ggplot(aes(x = Goldman_scaled_acc, color=Model)) + 
  geom_density(aes(fill= Model), alpha = 0.5) + 
  scale_color_manual(values=c("purple", "gold", "turquoise")) +
  scale_fill_manual(values=c("purple", "gold", "turquoise")) +
  theme_bw() + 
  ylab("Density") + 
  xlab("Goldman et al. Test Accuracy") + 
  ggtitle("Density of Goldman et al. Test Accuracy") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/accuracy_distribution.pdf", device="pdf", width = 8, height=5)
CHRTRM$Goldman_scaled_acc <- (CHRTRM$Goldman_acc/100 * 33 + .5) / 34

size_model = gam(Goldman_scaled_acc ~ log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)
summary(size_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.4786 -0.8088  0.0000  0.2771  1.6071 

(Dispersion Parameter for Beta regression family taken to be 0.3493)

    Null Deviance: -1.0739 on 33 degrees of freedom
Residual Deviance: 1.0581 on 32 degrees of freedom
AIC: -3694.535 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value   Pr(>F)    
log(Goldman_train_size)  1 19.689 19.6892   56.36 1.51e-08 ***
Residuals               32 11.179  0.3493                     
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit


lemma_model = gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CHRTRM)
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
summary(lemma_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = betar(link = "logit"), data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.9217 -0.8786  0.0000  0.0000  1.1945 

(Dispersion Parameter for Beta regression family taken to be 0.3355)

    Null Deviance: -1.0739 on 33 degrees of freedom
Residual Deviance: 12.528 on 32 degrees of freedom
AIC: -3798.311 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                          Df Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)  1 13.168 13.1679  39.247 5.054e-07 ***
Residuals                 32 10.736  0.3355                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
both_model <- gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)
summary(both_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.7181 -0.9309  0.0000  0.0000  1.3871 

(Dispersion Parameter for Beta regression family taken to be 0.3214)

    Null Deviance: -1.0739 on 33 degrees of freedom
Residual Deviance: 9.2018 on 31 degrees of freedom
AIC: -3878.832 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)  1 14.2329 14.2329 44.2893 1.932e-07 ***
log(Goldman_train_size)    1  0.7122  0.7122  2.2163    0.1467    
Residuals                 31  9.9622  0.3214                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
AIC(size_model)
[1] -3694.535
AIC(lemma_model)
[1] -3798.311
AIC(both_model)
[1] -3878.832
BIC(size_model)
[1] -3691.483
BIC(lemma_model)
[1] -3795.258
BIC(both_model)
[1] -3874.253
size_model$coefficients
            (Intercept) log(Goldman_train_size) 
             -4.9490885               0.6433317 
lemma_model$coefficients
              (Intercept) log(Goldman_train_lemmas) 
               -3.8351949                 0.8111991 
DeepSpin$Goldman_scaled_acc <- (DeepSpin$Goldman_acc/100 * 33 + .5) / 34

size_model = gam(Goldman_scaled_acc ~ log(Goldman_train_size), family=betar(link="logit"), data = DeepSpin)
res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
summary(size_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.5926 -1.0797  0.0000  0.0000  0.8346 

(Dispersion Parameter for Beta regression family taken to be 0.3122)

    Null Deviance: -19.4575 on 33 degrees of freedom
Residual Deviance: -12.9009 on 32 degrees of freedom
AIC: -12273.43 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value  Pr(>F)   
log(Goldman_train_size)  1 3.5977  3.5977  11.522 0.00185 **
Residuals               32 9.9918  0.3122                   
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit

lemma_model = gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)
res <- predict(lemma_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
summary(lemma_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = betar(link = "logit"), data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.6147 -1.0699  0.0000  0.0000  0.5571 

(Dispersion Parameter for Beta regression family taken to be 0.3817)

    Null Deviance: -19.4575 on 33 degrees of freedom
Residual Deviance: -17.41 on 32 degrees of freedom
AIC: -12163.35 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value  Pr(>F)  
log(Goldman_train_lemmas)  1  1.4442 1.44423  3.7841 0.06057 .
Residuals                 32 12.2131 0.38166                  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit

both_model <- gam(Goldman_scaled_acc ~ log(Goldman_train_size) + log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)
summary(both_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size) + 
    log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.6214 -0.9821  0.0000  0.0000  0.7569 

(Dispersion Parameter for Beta regression family taken to be 0.3064)

    Null Deviance: -19.4575 on 33 degrees of freedom
Residual Deviance: -12.6952 on 31 degrees of freedom
AIC: -12350.04 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df Sum Sq Mean Sq F value   Pr(>F)   
log(Goldman_train_size)    1 3.5682  3.5682 11.6462 0.001809 **
log(Goldman_train_lemmas)  1 0.8896  0.8896  2.9036 0.098388 . 
Residuals                 31 9.4980  0.3064                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
AIC(size_model)
[1] -12273.43
AIC(lemma_model)
[1] -12163.35
AIC(both_model)
[1] -12350.04
BIC(size_model)
[1] -12270.37
BIC(lemma_model)
[1] -12160.3
BIC(both_model)
[1] -12345.47
size_model$coefficients
            (Intercept) log(Goldman_train_size) 
             -0.6355997               0.2861804 
lemma_model$coefficients
              (Intercept) log(Goldman_train_lemmas) 
                0.6611814                 0.1920062 
CLUZH$Goldman_scaled_acc <- (CLUZH$Goldman_acc/100 * 33 + .5) / 34

size_model = gam(Goldman_scaled_acc ~ log(Goldman_train_size), family=betar(link="logit"), data = CLUZH)
res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit
summary(size_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.6764 -0.9117  0.0000  0.0000  0.7096 

(Dispersion Parameter for Beta regression family taken to be 0.4547)

    Null Deviance: -22.2953 on 33 degrees of freedom
Residual Deviance: -10.9053 on 32 degrees of freedom
AIC: -10671.66 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df  Sum Sq Mean Sq F value   Pr(>F)    
log(Goldman_train_size)  1  7.0202  7.0202   15.44 0.000427 ***
Residuals               32 14.5500  0.4547                     
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
lemma_model = gam(Goldman_scaled_acc ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CLUZH)
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
summary(lemma_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = betar(link = "logit"), data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.5490 -1.0794  0.0000  0.0000  0.8328 

(Dispersion Parameter for Beta regression family taken to be 0.5099)

    Null Deviance: -22.2953 on 33 degrees of freedom
Residual Deviance: -12.5557 on 32 degrees of freedom
AIC: -10473.82 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value   Pr(>F)   
log(Goldman_train_lemmas)  1  4.9185  4.9185  9.6466 0.003957 **
Residuals                 32 16.3160  0.5099                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
both_model <- gam(Goldman_scaled_acc ~ log(Goldman_train_size) + log(Goldman_train_lemmas), family=betar(link="logit"), data = CLUZH)
summary(both_model)

Call: gam(formula = Goldman_scaled_acc ~ log(Goldman_train_size) + 
    log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.6752 -0.8953  0.0000  0.0000  0.7093 

(Dispersion Parameter for Beta regression family taken to be 0.4671)

    Null Deviance: -22.2953 on 33 degrees of freedom
Residual Deviance: -11.0956 on 31 degrees of freedom
AIC: -10675.54 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_size)    1  7.0203  7.0203 15.0303 0.0005137 ***
log(Goldman_train_lemmas)  1  0.0050  0.0050  0.0107 0.9182216    
Residuals                 31 14.4794  0.4671                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
AIC(size_model)
[1] -10671.66
AIC(lemma_model)
[1] -10473.82
AIC(both_model)
[1] -10675.54
BIC(size_model)
[1] -10668.61
BIC(lemma_model)
[1] -10470.76
BIC(both_model)
[1] -10670.96
size_model$coefficients
            (Intercept) log(Goldman_train_size) 
             -1.7996408               0.3963434 
lemma_model$coefficients
              (Intercept) log(Goldman_train_lemmas) 
               -0.5208229                 0.3695156 
rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(Goldman_train_size, Goldman_scaled_acc)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(Goldman_train_size, size_fit)) + 
  geom_ribbon(aes(x = Goldman_train_size, 
                  ymin = size_fit - 2 * size_se,
                  ymax = size_fit + 2 * size_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Size") + 
  ylab("Accuracy") + 
  ggtitle("Training Size vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/train_size_beta.pdf", device="pdf", width = 12, height=4)
rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(Goldman_train_lemmas, Goldman_scaled_acc)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(Goldman_train_lemmas, lemma_fit)) + 
  geom_ribbon(aes(x = Goldman_train_lemmas, 
                  ymin = lemma_fit - 2 * lemma_se,
                  ymax = lemma_fit + 2 * lemma_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Lemmas") + 
  ylab("Accuracy") + 
  ggtitle("Training Lemmas vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


#ggsave("figures/train_lemmas_beta.pdf", device="pdf", width = 12, height=4)

Accuracy Drop

Initial Visualization

Raw Plots

all_data %>% 
  ggplot(aes(x = Goldman_train_size, y = acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Goldman et al. Training Size") + 
  ylab("Accuracy Drop") + 
  ggtitle("Training Size vs. Accuracy Drop") 

all_data %>% 
  ggplot(aes(x = Goldman_train_lemmas, y = acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Goldman et al. Training Lemmas") + 
  ylab("Accuracy Drop") + 
  ggtitle("Training Lemmas vs. Accuracy Drop")

all_data %>% 
  ggplot(aes(x = train_lemma_diff_raw, y = acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Drop in the Number of Training Lemmas") + 
  ylab("Accuracy Drop") + 
  ggtitle("Training Lemma Drop vs. Accuracy Drop")

Log-Log Plots and LM Fit Lines

all_data %>% 
  ggplot(aes(x = log(Goldman_train_size), y = log_acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Goldman et al. Training Size, Log Scale") + 
  ylab("Accuracy Drop, Log Scale") + 
  ggtitle("Training Size vs. Accuracy Drop") + 
  stat_smooth(method="lm", color="grey")

all_data %>% 
  ggplot(aes(x = log(Goldman_train_lemmas), y = log_acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Goldman et al. Training Lemmas, Log Scale") + 
  ylab("Accuracy Drop, Log Scale") + 
  ggtitle("Training Lemmas vs. Accuracy Drop") + 
  stat_smooth(method="lm", color="grey")

all_data %>% 
  ggplot(aes(x = log(-1*train_lemma_diff_raw), y = log_acc_drop)) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) +
  geom_point(aes(color=Family), size=3, alpha = 0.6) + 
  theme_bw() + 
  scale_color_manual(values=c("purple","turquoise", "gold")) +
  theme(plot.title = element_text(hjust=0.5, size=16)) +
  xlab("Drop in Training Lemmas, Log Scale") + 
  ylab("Accuracy Drop, Log Scale") + 
  ggtitle("Lemma Drop vs. Accuracy Drop") + 
  stat_smooth(method="lm", color="grey")

Basic Correlational Stats

CHRTRM$scaled_drop <- -1*(CHRTRM$Goldman_acc - CHRTRM$SIGMORPHON_acc - 2)/100
DeepSpin$scaled_drop <- -1*(DeepSpin$Goldman_acc - DeepSpin$SIGMORPHON_acc - 2)/100
CLUZH$scaled_drop <- -1*(CLUZH$Goldman_acc - CLUZH$SIGMORPHON_acc - 4)/100

correlations(log(CHRTRM$Goldman_train_size), log(CHRTRM$scaled_drop))
[1] "Pearson's product-moment correlation: -0.830738 (p = 0.000000)"
[1] "Spearman's rank correlation rho: -0.798319 (p = 0.000000)"
[1] "Kendall's rank correlation tau: -0.593583 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(DeepSpin$Goldman_train_size), log(DeepSpin$scaled_drop))
[1] "Pearson's product-moment correlation: -0.125834 (p = 0.478255)"
[1] "Spearman's rank correlation rho: -0.032631 (p = 0.854639)"
[1] "Kendall's rank correlation tau: -0.010841 (p = 0.928903)"
sprintf("")
[1] ""
correlations(log(CLUZH$Goldman_train_size), log(CLUZH$scaled_drop))
[1] "Pearson's product-moment correlation: -0.233565 (p = 0.183703)"
[1] "Spearman's rank correlation rho: -0.130491 (p = 0.461985)"
[1] "Kendall's rank correlation tau: -0.067797 (p = 0.573169)"
correlations(log(CHRTRM$Goldman_train_lemmas), log(CHRTRM$scaled_drop))
[1] "Pearson's product-moment correlation: -0.907756 (p = 0.000000)"
[1] "Spearman's rank correlation rho: -0.915412 (p = 0.000000)"
[1] "Kendall's rank correlation tau: -0.747098 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(DeepSpin$Goldman_train_lemmas), log(DeepSpin$scaled_drop))
[1] "Pearson's product-moment correlation: -0.095421 (p = 0.591397)"
[1] "Spearman's rank correlation rho: -0.080678 (p = 0.650137)"
[1] "Kendall's rank correlation tau: -0.052539 (p = 0.666183)"
sprintf("")
[1] ""
correlations(log(CLUZH$Goldman_train_lemmas), log(CLUZH$scaled_drop))
[1] "Pearson's product-moment correlation: -0.264233 (p = 0.131023)"
[1] "Spearman's rank correlation rho: -0.196087 (p = 0.266378)"
[1] "Kendall's rank correlation tau: -0.137746 (p = 0.253461)"
correlations(log(-1*CHRTRM$train_lemma_diff_raw), log(CHRTRM$scaled_drop))
[1] "Pearson's product-moment correlation: -0.896908 (p = 0.000000)"
[1] "Spearman's rank correlation rho: -0.888906 (p = 0.000000)"
[1] "Kendall's rank correlation tau: -0.726787 (p = 0.000000)"
sprintf("")
[1] ""
correlations(log(-1*DeepSpin$train_lemma_diff_raw), log(DeepSpin$scaled_drop))
[1] "Pearson's product-moment correlation: -0.146380 (p = 0.408762)"
[1] "Spearman's rank correlation rho: -0.128783 (p = 0.467918)"
[1] "Kendall's rank correlation tau: -0.086884 (p = 0.475256)"
sprintf("")
[1] ""
correlations(log(-1*CLUZH$train_lemma_diff_raw), log(CLUZH$scaled_drop))
[1] "Pearson's product-moment correlation: -0.281993 (p = 0.106145)"
[1] "Spearman's rank correlation rho: -0.236036 (p = 0.178977)"
[1] "Kendall's rank correlation tau: -0.157283 (p = 0.191899)"

Beta Regression

rbind(CLUZH, CHRTRM, DeepSpin) %>%
  ggplot(aes(x = scaled_drop, color=Model)) + 
  geom_density(aes(fill= Model), alpha = 0.5) + 
  scale_color_manual(values=c("purple", "gold", "turquoise")) +
  scale_fill_manual(values=c("purple", "gold", "turquoise")) +
  theme_bw() + 
  ylab("Density") + 
  xlab("Accuracy Drop") + 
  ggtitle("Density of Accuracy Drop") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 

ggsave("figures/drop_distribution.pdf", device="pdf", width = 8, height=5)

size_model = gam(scaled_drop ~ log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)

summary(size_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.3926 -0.3985  0.0000  0.8903  1.0690 

(Dispersion Parameter for Beta regression family taken to be 0.3693)

    Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: -3.3233 on 32 degrees of freedom
AIC: 1550.002 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_size)  1 15.365 15.3647   41.61 2.964e-07 ***
Residuals               32 11.816  0.3693                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type="response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit

lemma_model = gam(scaled_drop ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CHRTRM)

summary(lemma_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.1012  0.0000  0.6957  1.0132  1.9912 

(Dispersion Parameter for Beta regression family taken to be 0.1745)

    Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: 16.7735 on 32 degrees of freedom
AIC: 1016.286 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                          Df  Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)  1 11.8181 11.8181  67.719 2.119e-09 ***
Residuals                 32  5.5846  0.1745                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type="response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit

drop_model = gam(scaled_drop ~ log(-1*train_lemma_diff_raw), family=betar(link="logit"), data = CHRTRM)

summary(drop_model)

Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.5036  0.0000  0.4835  0.9504  1.5863 

(Dispersion Parameter for Beta regression family taken to be 0.2471)

    Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: 8.804 on 32 degrees of freedom
AIC: 1193.401 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                               Df  Sum Sq Mean Sq F value    Pr(>F)    
log(-1 * train_lemma_diff_raw)  1 12.6416 12.6416  51.154 4.056e-08 ***
Residuals                      32  7.9081  0.2471                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(drop_model, newdata = CHRTRM, se.fit = TRUE, type="response")
CHRTRM$drop_fit <- res$fit
CHRTRM$drop_se <- res$se.fit

both_model <- gam(scaled_drop ~ log(Goldman_train_lemmas)  + log(-1* train_lemma_diff_raw) + log(Goldman_train_size), family=betar(link="logit"), data = CHRTRM)

summary(both_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas) + log(-1 * 
    train_lemma_diff_raw) + log(Goldman_train_size), family = betar(link = "logit"), 
    data = CHRTRM)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-1.0309  0.0000  0.6883  1.1058  2.1611 

(Dispersion Parameter for Beta regression family taken to be 0.1944)

    Null Deviance: -4.2629 on 33 degrees of freedom
Residual Deviance: 20.2587 on 30 degrees of freedom
AIC: 956.8627 

Number of Local Scoring Iterations: 6 

Anova for Parametric Effects
                               Df  Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)       1 10.9621 10.9621 56.3870 2.267e-08 ***
log(-1 * train_lemma_diff_raw)  1  0.2024  0.2024  1.0412    0.3157    
log(Goldman_train_size)         1  0.1658  0.1658  0.8530    0.3631    
Residuals                      30  5.8323  0.1944                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model$coefficients
            (Intercept) log(Goldman_train_size) 
              3.9535916              -0.5549763 
lemma_model$coefficients
              (Intercept) log(Goldman_train_lemmas) 
                3.9757535                -0.9047767 
drop_model$coefficients
                   (Intercept) log(-1 * train_lemma_diff_raw) 
                     2.8158287                     -0.8161564 
AIC(size_model)
[1] 1550.002
AIC(lemma_model)
[1] 1016.286
AIC(drop_model)
[1] 1193.401
BIC(size_model)
[1] 1553.055
BIC(lemma_model)
[1] 1019.338
BIC(drop_model)
[1] 1196.454
size_model = gam(scaled_drop ~ log(Goldman_train_size), family=betar(link="logit"), data = DeepSpin)

summary(size_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.7397  0.0000  0.0000  1.1084  1.4802 

(Dispersion Parameter for Beta regression family taken to be 0.1433)

    Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -12.4806 on 32 degrees of freedom
AIC: 7018.613 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value   Pr(>F)   
log(Goldman_train_size)  1 1.8557 1.85567  12.945 0.001067 **
Residuals               32 4.5871 0.14335                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type="response")
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit

lemma_model = gam(scaled_drop ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)

summary(lemma_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.5064  0.0000  0.0000  1.0176  1.5635 

(Dispersion Parameter for Beta regression family taken to be 0.1553)

    Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -14.7622 on 32 degrees of freedom
AIC: 7027.519 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df Sum Sq Mean Sq F value   Pr(>F)   
log(Goldman_train_lemmas)  1 1.1735 1.17353  7.5547 0.009755 **
Residuals                 32 4.9708 0.15534                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(lemma_model, newdata = DeepSpin, se.fit = TRUE, type="response")
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit

drop_model = gam(scaled_drop ~ log(-1*train_lemma_diff_raw), family=betar(link="logit"), data = DeepSpin)

summary(drop_model)

Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.5698  0.0000  0.0000  1.0924  1.5544 

(Dispersion Parameter for Beta regression family taken to be 0.1475)

    Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -12.6903 on 32 degrees of freedom
AIC: 6992.3 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                               Df Sum Sq Mean Sq F value   Pr(>F)   
log(-1 * train_lemma_diff_raw)  1 1.5505  1.5505  10.512 0.002771 **
Residuals                      32 4.7199  0.1475                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(drop_model, newdata = DeepSpin, se.fit = TRUE, type="response")
DeepSpin$drop_fit <- res$fit
DeepSpin$drop_se <- res$se.fit

both_model <- gam(scaled_drop ~ log(-1*train_lemma_diff_raw) + log(Goldman_train_size) + log(Goldman_train_lemmas), family=betar(link="logit"), data = DeepSpin)

summary(both_model)

Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw) + 
    log(Goldman_train_size) + log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = DeepSpin)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.8752  0.0000  0.0000  0.9393  1.5220 

(Dispersion Parameter for Beta regression family taken to be 0.1435)

    Null Deviance: -16.5461 on 33 degrees of freedom
Residual Deviance: -12.7918 on 30 degrees of freedom
AIC: 6982.896 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                               Df Sum Sq Mean Sq F value   Pr(>F)   
log(-1 * train_lemma_diff_raw)  1 1.6345 1.63447 11.3933 0.002052 **
log(Goldman_train_size)         1 0.1767 0.17666  1.2314 0.275945   
log(Goldman_train_lemmas)       1 0.3227 0.32272  2.2495 0.144106   
Residuals                      30 4.3038 0.14346                    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model$coefficients
            (Intercept) log(Goldman_train_size) 
             -0.2192908              -0.2378525 
lemma_model$coefficients
              (Intercept) log(Goldman_train_lemmas) 
               -1.0371551                -0.2098211 
drop_model$coefficients
                   (Intercept) log(-1 * train_lemma_diff_raw) 
                    -1.0666528                     -0.2533969 
AIC(size_model)
[1] 7018.613
AIC(lemma_model)
[1] 7027.519
AIC(drop_model)
[1] 6992.3
BIC(size_model)
[1] 7021.666
BIC(lemma_model)
[1] 7030.572
BIC(drop_model)
[1] 6995.352
size_model = gam(scaled_drop ~ log(Goldman_train_size), family=betar(link="logit"), data = CLUZH)

summary(size_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_size), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
   Min     1Q Median     3Q    Max 
-0.838  0.000  0.000  1.005  1.417 

(Dispersion Parameter for Beta regression family taken to be 0.1677)

    Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 1.805 on 32 degrees of freedom
AIC: 5855.958 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                        Df Sum Sq Mean Sq F value   Pr(>F)    
log(Goldman_train_size)  1 2.2528 2.25278  13.431 0.000889 ***
Residuals               32 5.3675 0.16774                     
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type="response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit

lemma_model = gam(scaled_drop ~ log(Goldman_train_lemmas), family=betar(link="logit"), data = CLUZH)

summary(lemma_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.6502  0.0000  0.3396  1.0907  1.4213 

(Dispersion Parameter for Beta regression family taken to be 0.1501)

    Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 3.7868 on 32 degrees of freedom
AIC: 5823.695 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                          Df Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)  1 2.5131 2.51310  16.738 0.0002709 ***
Residuals                 32 4.8047 0.15015                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type="response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit

drop_model = gam(scaled_drop ~ log(-1*train_lemma_diff_raw), family=betar(link="logit"), data = CLUZH)

summary(drop_model)

Call: gam(formula = scaled_drop ~ log(-1 * train_lemma_diff_raw), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
   Min     1Q Median     3Q    Max 
-0.841  0.000  0.000  1.094  1.376 

(Dispersion Parameter for Beta regression family taken to be 0.1516)

    Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 3.8613 on 32 degrees of freedom
AIC: 5831.558 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                               Df Sum Sq Mean Sq F value    Pr(>F)    
log(-1 * train_lemma_diff_raw)  1 2.5820 2.58196  17.027 0.0002453 ***
Residuals                      32 4.8526 0.15164                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
res <- predict(drop_model, newdata = CLUZH, se.fit = TRUE, type="response")
CLUZH$drop_fit <- res$fit
CLUZH$drop_se <- res$se.fit

both_model <- gam(scaled_drop ~ log(Goldman_train_lemmas) + log(-1*train_lemma_diff_raw) + log(Goldman_train_size) , family=betar(link="logit"), data = CLUZH)

summary(both_model)

Call: gam(formula = scaled_drop ~ log(Goldman_train_lemmas) + log(-1 * 
    train_lemma_diff_raw) + log(Goldman_train_size), family = betar(link = "logit"), 
    data = CLUZH)
Deviance Residuals:
    Min      1Q  Median      3Q     Max 
-0.7097  0.0000  0.3350  1.0758  1.4194 

(Dispersion Parameter for Beta regression family taken to be 0.1584)

    Null Deviance: -3.114 on 33 degrees of freedom
Residual Deviance: 4.0005 on 30 degrees of freedom
AIC: 5837.224 

Number of Local Scoring Iterations: 2 

Anova for Parametric Effects
                               Df Sum Sq Mean Sq F value    Pr(>F)    
log(Goldman_train_lemmas)       1 2.5226 2.52261 15.9285 0.0003913 ***
log(-1 * train_lemma_diff_raw)  1 0.0524 0.05242  0.3310 0.5693799    
log(Goldman_train_size)         1 0.0128 0.01279  0.0808 0.7782086    
Residuals                      30 4.7511 0.15837                      
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model$coefficients
            (Intercept) log(Goldman_train_size) 
              0.1286658              -0.2387694 
lemma_model$coefficients
              (Intercept) log(Goldman_train_lemmas) 
                -0.328039                 -0.287939 
drop_model$coefficients
                   (Intercept) log(-1 * train_lemma_diff_raw) 
                    -0.5445538                     -0.3002098 
AIC(size_model)
[1] 5855.958
AIC(lemma_model)
[1] 5823.695
AIC(drop_model)
[1] 5831.558
BIC(size_model)
[1] 5859.011
BIC(lemma_model)
[1] 5826.747
BIC(drop_model)
[1] 5834.611
rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(Goldman_train_size, -1*scaled_drop)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(Goldman_train_size, -1*size_fit)) + 
  geom_ribbon(aes(x = Goldman_train_size, 
                  ymin = -1*(size_fit - 2 * size_se),
                  ymax = -1*(size_fit + 2 * size_se),
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Size") + 
  ylab("Scaled Accuracy Drop") + 
  ggtitle("Training Size vs. Accuracy Drop") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/train_size_drop.pdf", device="pdf", width = 12, height=4)
rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(Goldman_train_lemmas, -1*scaled_drop)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(Goldman_train_lemmas, -1*lemma_fit)) + 
  geom_ribbon(aes(x = Goldman_train_lemmas, 
                  ymin = -1*(lemma_fit - 2 * lemma_se),
                  ymax = -1*(lemma_fit + 2 * lemma_se),
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Lemmas") + 
  ylab("Scaled Accuracy Drop") +
  ggtitle("Training Lemmas vs. Accuracy Drop") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/train_lemmas_drop.pdf", device="pdf", width = 12, height=4)

rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(-1*train_lemma_diff_raw, -1*scaled_drop)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(-1*train_lemma_diff_raw, -1*drop_fit)) + 
  geom_ribbon(aes(x = -1*train_lemma_diff_raw, 
                  ymin = -1*(drop_fit - 2 * drop_se),
                  ymax = -1*(drop_fit + 2 * drop_se),
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Drop in Lemmas from SIGMORPHON to Goldman et al.") + 
  ylab("Scaled Accuracy Drop") +
  ggtitle("Training Lemma Drop vs. Accuracy Drop") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/lemma_drop_drop.pdf", device="pdf", width = 12, height=4)

Other Regressions

Linear Regression: Raw Accuracy

size_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_size),  data = CHRTRM)
summary(size_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_size), 
    data = CHRTRM)

Residuals:
     Min       1Q   Median       3Q      Max 
-2.12468 -0.31146  0.01354  0.48987  1.86432 

Coefficients:
                        Estimate Std. Error t value Pr(>|t|)    
(Intercept)             -4.58098    0.65386  -7.006 6.12e-08 ***
log(Goldman_train_size)  0.42338    0.07346   5.763 2.16e-06 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9212 on 32 degrees of freedom
Multiple R-squared:  0.5093,    Adjusted R-squared:  0.494 
F-statistic: 33.22 on 1 and 32 DF,  p-value: 2.156e-06
res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit


lemma_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), data = CHRTRM)
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
summary(lemma_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), 
    data = CHRTRM)

Residuals:
    Min      1Q  Median      3Q     Max 
-2.5247 -0.5142  0.3007  0.7480  1.1648 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)    
(Intercept)               -3.21974    0.47613  -6.762 1.22e-07 ***
log(Goldman_train_lemmas)  0.40297    0.07828   5.148 1.29e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9727 on 32 degrees of freedom
Multiple R-squared:  0.453, Adjusted R-squared:  0.4359 
F-statistic:  26.5 on 1 and 32 DF,  p-value: 1.293e-05
both_model <- lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CHRTRM)
summary(both_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), data = CHRTRM)

Residuals:
     Min       1Q   Median       3Q      Max 
-2.18137 -0.36728  0.06201  0.54080  1.74301 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)    
(Intercept)                -4.3744     0.7339  -5.960 1.37e-06 ***
log(Goldman_train_lemmas)   0.1066     0.1656   0.644   0.5246    
log(Goldman_train_size)     0.3292     0.1641   2.006   0.0537 .  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.9298 on 31 degrees of freedom
Multiple R-squared:  0.5158,    Adjusted R-squared:  0.4846 
F-statistic: 16.51 on 2 and 31 DF,  p-value: 1.312e-05
AIC(size_model)
[1] 94.84671
AIC(lemma_model)
[1] 98.5432
AIC(both_model)
[1] 96.39553
anova(both_model, lemma_model)
Analysis of Variance Table

Model 1: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas)
  Res.Df    RSS Df Sum of Sq     F  Pr(>F)  
1     31 26.799                             
2     32 30.276 -1   -3.4769 4.022 0.05371 .
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_size),  data = DeepSpin)
summary(size_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_size), 
    data = DeepSpin)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.09702 -0.07001  0.01215  0.18962  0.42460 

Coefficients:
                        Estimate Std. Error t value Pr(>|t|)   
(Intercept)             -0.77218    0.22958  -3.363  0.00201 **
log(Goldman_train_size)  0.06590    0.02579   2.555  0.01558 * 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.3234 on 32 degrees of freedom
Multiple R-squared:  0.1694,    Adjusted R-squared:  0.1435 
F-statistic: 6.527 on 1 and 32 DF,  p-value: 0.01558
res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit


lemma_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), data = DeepSpin)
res <- predict(lemma_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit
summary(lemma_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), 
    data = DeepSpin)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.28791 -0.03124  0.06302  0.20218  0.30658 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)  
(Intercept)               -0.42363    0.16866  -2.512   0.0173 *
log(Goldman_train_lemmas)  0.03873    0.02773   1.396   0.1722  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.3446 on 32 degrees of freedom
Multiple R-squared:  0.05744,   Adjusted R-squared:  0.02799 
F-statistic:  1.95 on 1 and 32 DF,  p-value: 0.1722
both_model <- lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = DeepSpin)
summary(both_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), data = DeepSpin)

Residuals:
     Min       1Q   Median       3Q      Max 
-0.89596 -0.05727  0.06613  0.13517  0.53952 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)    
(Intercept)               -0.96792    0.24663  -3.925  0.00045 ***
log(Goldman_train_lemmas) -0.10099    0.05567  -1.814  0.07936 .  
log(Goldman_train_size)    0.15516    0.05516   2.813  0.00844 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.3125 on 31 degrees of freedom
Multiple R-squared:  0.2491,    Adjusted R-squared:  0.2007 
F-statistic: 5.143 on 2 and 31 DF,  p-value: 0.01178
AIC(size_model)
[1] 23.67421
AIC(lemma_model)
[1] 27.97414
AIC(both_model)
[1] 22.24389
anova(both_model, lemma_model)
Analysis of Variance Table

Model 1: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas)
  Res.Df    RSS Df Sum of Sq      F  Pr(>F)   
1     31 3.0265                               
2     32 3.7992 -1  -0.77262 7.9137 0.00844 **
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
size_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_size),  data = CLUZH)
summary(size_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_size), 
    data = CLUZH)

Residuals:
     Min       1Q   Median       3Q      Max 
-2.01946 -0.16571 -0.01182  0.38359  0.86925 

Coefficients:
                        Estimate Std. Error t value Pr(>|t|)    
(Intercept)             -1.65335    0.40383  -4.094 0.000269 ***
log(Goldman_train_size)  0.15234    0.04537   3.358 0.002041 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.5689 on 32 degrees of freedom
Multiple R-squared:  0.2605,    Adjusted R-squared:  0.2374 
F-statistic: 11.27 on 1 and 32 DF,  p-value: 0.002041
res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit


lemma_model = lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), data = CLUZH)
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
summary(lemma_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas), 
    data = CLUZH)

Residuals:
    Min      1Q  Median      3Q     Max 
-2.4300 -0.1431  0.1326  0.3918  0.6468 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)   
(Intercept)               -0.99545    0.29911  -3.328  0.00221 **
log(Goldman_train_lemmas)  0.11548    0.04918   2.348  0.02521 * 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.6111 on 32 degrees of freedom
Multiple R-squared:  0.147, Adjusted R-squared:  0.1203 
F-statistic: 5.514 on 1 and 32 DF,  p-value: 0.02521
both_model <- lm(log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CLUZH)
summary(both_model)

Call:
lm(formula = log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), data = CLUZH)

Residuals:
     Min       1Q   Median       3Q      Max 
-1.80791 -0.16141  0.01489  0.28101  0.99017 

Coefficients:
                          Estimate Std. Error t value Pr(>|t|)    
(Intercept)                -1.8593     0.4484  -4.147 0.000242 ***
log(Goldman_train_lemmas)  -0.1063     0.1012  -1.050 0.301858    
log(Goldman_train_size)     0.2463     0.1003   2.456 0.019855 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.568 on 31 degrees of freedom
Multiple R-squared:  0.2859,    Adjusted R-squared:  0.2399 
F-statistic: 6.206 on 2 and 31 DF,  p-value: 0.005408
AIC(size_model)
[1] 62.07644
AIC(lemma_model)
[1] 66.93316
AIC(both_model)
[1] 62.88837
anova(both_model, lemma_model)
Analysis of Variance Table

Model 1: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: log(Goldman_scaled_acc) ~ log(Goldman_train_lemmas)
  Res.Df    RSS Df Sum of Sq      F  Pr(>F)  
1     31 10.003                              
2     32 11.949 -1   -1.9462 6.0317 0.01986 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(log(Goldman_train_size), log(Goldman_scaled_acc))) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(log(Goldman_train_size), size_fit)) + 
  geom_ribbon(aes(x = log(Goldman_train_size), 
                  ymin = size_fit - 2 * size_se,
                  ymax = size_fit + 2 * size_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Size, Log Scale") + 
  ylab("Accuracy") + 
  ggtitle("Training Size vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/train_size_lm.pdf", device="pdf", width = 12, height=4)
rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(log(Goldman_train_lemmas), log(Goldman_scaled_acc))) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(log(Goldman_train_lemmas), lemma_fit)) + 
  geom_ribbon(aes(x = log(Goldman_train_lemmas), 
                  ymin = lemma_fit - 2 * lemma_se,
                  ymax = lemma_fit + 2 * lemma_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Lemmas, Log Scale") + 
  ylab("Accuracy") + 
  ggtitle("Training Lemmas vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/train_lemmas_lm.pdf", device="pdf", width = 12, height=4)

Logistic Regression: Raw Accuracy

size_model = glm(Goldman_scaled_acc ~ log(Goldman_train_size),  data = CHRTRM, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(size_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = binomial, 
    data = CHRTRM)

Coefficients:
                        Estimate Std. Error z value Pr(>|z|)   
(Intercept)              -5.0645     1.9139  -2.646  0.00814 **
log(Goldman_train_size)   0.6567     0.2284   2.875  0.00404 **
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 18.0726  on 33  degrees of freedom
Residual deviance:  6.3867  on 32  degrees of freedom
AIC: 31.886

Number of Fisher Scoring iterations: 5
res <- predict(size_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$size_fit <- res$fit
CHRTRM$size_se <- res$se.fit


lemma_model = glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas), data = CHRTRM, family=binomial)
Warning: non-integer #successes in a binomial glm!
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
CHRTRM$lemma_fit <- res$fit
CHRTRM$lemma_se <- res$se.fit
summary(lemma_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = binomial, data = CHRTRM)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)  
(Intercept)                -3.8352     1.6005  -2.396   0.0166 *
log(Goldman_train_lemmas)   0.8112     0.3161   2.566   0.0103 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 18.0726  on 33  degrees of freedom
Residual deviance:  5.6775  on 32  degrees of freedom
AIC: 28.439

Number of Fisher Scoring iterations: 5
both_model <- glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CHRTRM, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(both_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), family = binomial, data = CHRTRM)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)  
(Intercept)                -4.5197     1.9676  -2.297   0.0216 *
log(Goldman_train_lemmas)   0.5324     0.5415   0.983   0.3255  
log(Goldman_train_size)     0.2584     0.4329   0.597   0.5507  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 18.0726  on 33  degrees of freedom
Residual deviance:  5.3129  on 31  degrees of freedom
AIC: 30.767

Number of Fisher Scoring iterations: 5
AIC(size_model)
[1] 31.88575
AIC(lemma_model)
[1] 28.4393
AIC(both_model)
[1] 30.76683
anova(both_model, lemma_model)
Analysis of Deviance Table

Model 1: Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: Goldman_scaled_acc ~ log(Goldman_train_lemmas)
  Resid. Df Resid. Dev Df Deviance Pr(>Chi)
1        31     5.3129                     
2        32     5.6775 -1 -0.36453    0.546
size_model = glm(Goldman_scaled_acc ~ log(Goldman_train_size),  data = DeepSpin, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(size_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = binomial, 
    data = DeepSpin)

Coefficients:
                        Estimate Std. Error z value Pr(>|z|)
(Intercept)              -0.5417     1.9087  -0.284    0.777
log(Goldman_train_size)   0.2811     0.2380   1.181    0.238

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 8.0760  on 33  degrees of freedom
Residual deviance: 6.5801  on 32  degrees of freedom
AIC: 19.053

Number of Fisher Scoring iterations: 5
res <- predict(size_model, newdata = DeepSpin, se.fit = TRUE, type = "response")
DeepSpin$size_fit <- res$fit
DeepSpin$size_se <- res$se.fit


lemma_model = glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas), data = DeepSpin, family=binomial)
Warning: non-integer #successes in a binomial glm!
res <- predict(lemma_model, newdata = CHRTRM, se.fit = TRUE, type = "response")
DeepSpin$lemma_fit <- res$fit
DeepSpin$lemma_se <- res$se.fit
summary(lemma_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = binomial, data = DeepSpin)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)
(Intercept)                 0.7911     1.3545   0.584    0.559
log(Goldman_train_lemmas)   0.1786     0.2453   0.728    0.466

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 8.0760  on 33  degrees of freedom
Residual deviance: 7.5112  on 32  degrees of freedom
AIC: 21.031

Number of Fisher Scoring iterations: 5
both_model <- glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = DeepSpin, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(both_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), family = binomial, data = DeepSpin)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)
(Intercept)                -0.9913     1.9715  -0.503    0.615
log(Goldman_train_lemmas)  -0.3398     0.5003  -0.679    0.497
log(Goldman_train_size)     0.5596     0.4763   1.175    0.240

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 8.0760  on 33  degrees of freedom
Residual deviance: 6.1213  on 31  degrees of freedom
AIC: 19.851

Number of Fisher Scoring iterations: 5
AIC(size_model)
[1] 19.05313
AIC(lemma_model)
[1] 21.03097
AIC(both_model)
[1] 19.85117
anova(both_model, lemma_model)
Analysis of Deviance Table

Model 1: Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: Goldman_scaled_acc ~ log(Goldman_train_lemmas)
  Resid. Df Resid. Dev Df Deviance Pr(>Chi)
1        31     6.1213                     
2        32     7.5112 -1    -1.39   0.2384
size_model = glm(Goldman_scaled_acc ~ log(Goldman_train_size),  data = CLUZH, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(size_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_size), family = binomial, 
    data = CLUZH)

Coefficients:
                        Estimate Std. Error z value Pr(>|z|)  
(Intercept)              -1.8475     1.8112  -1.020   0.3077  
log(Goldman_train_size)   0.4071     0.2317   1.757   0.0789 .
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 12.9162  on 33  degrees of freedom
Residual deviance:  9.3231  on 32  degrees of freedom
AIC: 24.943

Number of Fisher Scoring iterations: 5
res <- predict(size_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$size_fit <- res$fit
CLUZH$size_se <- res$se.fit


lemma_model = glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas), data = CLUZH, family=binomial)
Warning: non-integer #successes in a binomial glm!
res <- predict(lemma_model, newdata = CLUZH, se.fit = TRUE, type = "response")
CLUZH$lemma_fit <- res$fit
CLUZH$lemma_se <- res$se.fit
summary(lemma_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas), 
    family = binomial, data = CLUZH)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)
(Intercept)                -0.5165     1.3129  -0.393    0.694
log(Goldman_train_lemmas)   0.3758     0.2575   1.460    0.144

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 12.916  on 33  degrees of freedom
Residual deviance: 10.291  on 32  degrees of freedom
AIC: 26.417

Number of Fisher Scoring iterations: 5
both_model <- glm(Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size), data = CLUZH, family=binomial)
Warning: non-integer #successes in a binomial glm!
summary(both_model)

Call:
glm(formula = Goldman_scaled_acc ~ log(Goldman_train_lemmas) + 
    log(Goldman_train_size), family = binomial, data = CLUZH)

Coefficients:
                          Estimate Std. Error z value Pr(>|z|)
(Intercept)               -1.89248    1.89135  -1.001    0.317
log(Goldman_train_lemmas) -0.03862    0.48110  -0.080    0.936
log(Goldman_train_size)    0.43737    0.44328   0.987    0.324

(Dispersion parameter for binomial family taken to be 1)

    Null deviance: 12.9162  on 33  degrees of freedom
Residual deviance:  9.3167  on 31  degrees of freedom
AIC: 27.012

Number of Fisher Scoring iterations: 5
AIC(size_model)
[1] 24.94273
AIC(lemma_model)
[1] 26.41689
AIC(both_model)
[1] 27.01204
anova(both_model, lemma_model)
Analysis of Deviance Table

Model 1: Goldman_scaled_acc ~ log(Goldman_train_lemmas) + log(Goldman_train_size)
Model 2: Goldman_scaled_acc ~ log(Goldman_train_lemmas)
  Resid. Df Resid. Dev Df Deviance Pr(>Chi)
1        31     9.3167                     
2        32    10.2914 -1 -0.97468   0.3235
rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(log(Goldman_train_size), Goldman_scaled_acc)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(log(Goldman_train_size), size_fit)) + 
  geom_ribbon(aes(x = log(Goldman_train_size), 
                  ymin = size_fit - 2 * size_se,
                  ymax = size_fit + 2 * size_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Size, Log Scale") + 
  ylab("Accuracy") + 
  ggtitle("Training Size vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/train_size_logistic.pdf", device="pdf", width = 12, height=4)
rbind(CLUZH, CHRTRM, DeepSpin) %>% 
  ggplot(aes(log(Goldman_train_lemmas), Goldman_scaled_acc)) + 
  geom_point(aes(color=Family), size = 3, alpha = 0.6) + 
  scale_color_manual(values=c("turquoise", "purple", "gold")) + 
  facet_grid(~factor(Model, levels=c("CHR-TRM", "DeepSpin", "CLUZH"))) + 
  geom_line(aes(log(Goldman_train_lemmas), lemma_fit)) + 
  geom_ribbon(aes(x = log(Goldman_train_lemmas), 
                  ymin = lemma_fit - 2 * lemma_se,
                  ymax = lemma_fit + 2 * lemma_se,
                  ), fill ="grey",
                  alpha = 0.4) + 
  theme_bw() + 
  xlab("Goldman Training Lemmas, Log Scale") + 
  ylab("Accuracy") + 
  ggtitle("Training Lemmas vs. Accuracy, Goldman et al. Data") + 
  theme(plot.title = element_text(hjust=0.5, size=16)) 


ggsave("figures/train_lemmas_logistic.pdf", device="pdf", width = 12, height=4)
LS0tCnRpdGxlOiAiUiBOb3RlYm9vayIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKIyBSZXBsaWNhdGlvbiBvZiBHb2xkbWFuIGV0IGFsLgoKYGBge3J9CmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeSh0aWR5dmVyc2UpCmxpYnJhcnkoY2FyKQpsaWJyYXJ5KG1nY3YpCnJlcXVpcmUoZ2FtKQpyZXF1aXJlKGxtdGVzdCkKcmVxdWlyZShsbWU0KQpgYGAKCiMjIExvYWQgaW4gJiBQcmVwcm9jZXNzIHRoZSBEYXRhCgpgYGB7cn0KZGY8LXJlYWQuY3N2KCJyZXBsaWNhdGlvbl9maW5hbC5jc3YiKQpkZiRGYW1pbHkgPSBzdHJfdG9fdGl0bGUoc3RyX3JlcGxhY2UoZGYkRmFtaWx5LCAiXyIsICIgIikpCgpDSFJUUk0gPC0gZGYgJT4lIAogIHNlbGVjdChjKExhbmd1YWdlLAogICAgICAgICAgIEZhbWlseSwKICAgICAgICAgICBTSUdNT1JQSE9OX2FjYyA9IENIUi5UUk1fU0lHTU9SUEhPTl90ZXN0X2FjYywgCiAgICAgICAgICAgR29sZG1hbl9hY2MgPSBDSFIuVFJNX0dvbGRtYW5fdGVzdF9hY2MsIAogICAgICAgICAgIEdvbGRtYW5fdHJhaW5fc2l6ZSwgCiAgICAgICAgICAgR29sZG1hbl90cmFpbl9sZW1tYXMsCiAgICAgICAgICAgdHJhaW5fbGVtbWFfZGlmZl9yYXcsIAogICAgICAgICAgIFNJR01PUlBIT05fdHJhaW5fc2l6ZSwgCiAgICAgICAgICAgU0lHTU9SUEhPTl90cmFpbl9sZW1tYXMKICAgICAgICAgICApKQoKQ0hSVFJNJE1vZGVsID0gIkNIUi1UUk0iCgoKCkNMVVpIIDwtIGRmICU+JSAKICBzZWxlY3QoYyhMYW5ndWFnZSwKICAgICAgICAgICBGYW1pbHksCiAgICAgICAgICAgU0lHTU9SUEhPTl9hY2MgPSBDTFVaSF9TSUdNT1JQSE9OX3Rlc3RfYWNjLCAKICAgICAgICAgICBHb2xkbWFuX2FjYyA9IENMVVpIX0dvbGRtYW5fdGVzdF9hY2MsIAogICAgICAgICAgIEdvbGRtYW5fdHJhaW5fc2l6ZSwgCiAgICAgICAgICAgR29sZG1hbl90cmFpbl9sZW1tYXMsCiAgICAgICAgICAgdHJhaW5fbGVtbWFfZGlmZl9yYXcsCiAgICAgICAgICAgU0lHTU9SUEhPTl90cmFpbl9zaXplLAogICAgICAgICAgIFNJR01PUlBIT05fdHJhaW5fbGVtbWFzCiAgICAgICAgICAgKSkKCgpDTFVaSCRNb2RlbCA9ICJDTFVaSCIKCgoKRGVlcFNwaW4gPC0gZGYgJT4lIAogIHNlbGVjdChjKExhbmd1YWdlLAogICAgICAgICAgIEZhbWlseSwKICAgICAgICAgICBTSUdNT1JQSE9OX2FjYyA9IERlZXBTcGluX1NJR01PUlBIT05fdGVzdF9hY2MsIAogICAgICAgICAgIEdvbGRtYW5fYWNjID0gRGVlcFNwaW5fR29sZG1hbl90ZXN0X2FjYywgCiAgICAgICAgICAgR29sZG1hbl90cmFpbl9zaXplLCAKICAgICAgICAgICBHb2xkbWFuX3RyYWluX2xlbW1hcywKICAgICAgICAgICB0cmFpbl9sZW1tYV9kaWZmX3JhdywKICAgICAgICAgICBTSUdNT1JQSE9OX3RyYWluX3NpemUsCiAgICAgICAgICAgU0lHTU9SUEhPTl90cmFpbl9sZW1tYXMKICAgICAgICAgICApKQoKRGVlcFNwaW4kTW9kZWwgPSAiRGVlcFNwaW4iCgoKCmFsbF9kYXRhIDwtIHJiaW5kKENMVVpILCBDSFJUUk0sIERlZXBTcGluKQoKCgphbGxfZGF0YSRhY2NfZHJvcCA9IGFsbF9kYXRhJEdvbGRtYW5fYWNjIC0gYWxsX2RhdGEkU0lHTU9SUEhPTl9hY2MKYWxsX2RhdGEkbG9nX2FjY19kcm9wIDwtIC0xKmxvZygtMSooYWxsX2RhdGEkYWNjX2Ryb3AgLSA0KSkKbWF4KGFsbF9kYXRhJGFjY19kcm9wKQpgYGAKCmBgYHtyfQojIFdlJ2xsIHVzZSB0aGlzIGhlbHBlciBmdW5jdGlvbiB0byBydW4gb3VyIHN0YXRzCmNvcnJlbGF0aW9ucyA8LSBmdW5jdGlvbihhLCBiKXsKICBmb3IgKG0gaW4gbGlzdCgicGVhcnNvbiIsICJzcGVhcm1hbiIsICJrZW5kYWxsIikpewogICAgIyBTdXByZXNzaW5nIHdhcm5pbmdzIGJjIHdlJ2xsIGdldCB0aGVtIHdoZW5ldmVyIHRoZXJlIGFyZSB0aWVzIAogICAgc3VwcHJlc3NXYXJuaW5ncyhyZXMgPC0gY29yLnRlc3QoYSwgYiwgbWV0aG9kID0gbSkpCiAgICBmb3JtYXR0ZWQgPC0gc3ByaW50ZigiJXM6ICVmIChwID0gJWYpIiwgcmVzJG1ldGhvZCwgcmVzJGVzdGltYXRlLCByZXMkcC52YWx1ZSkKICAgIHByaW50KGZvcm1hdHRlZCkKICB9Cn0KCiMgV2UnbGwgdXNlIHRoaXMgaGVscGVyIGZ1bmN0aW9uIHRvIGdldCBpbmZvcm1hdGlvbiBhYm91dCBvdXIgbW9kZWwKZXZhbF9tb2RlbCA8LSBmdW5jdGlvbihtb2RlbCwgZGYpewogIHJzcXVhcmVkID0gc3VtbWFyeShtb2RlbCkkci5zcXVhcmVkCiAgQUlDID0gQUlDKG1vZGVsKQogIHJlc3VsdHMgPC0gc3ByaW50ZigiUl4yOiAlZiwgQUlDOiAlZiIsIHJzcXVhcmVkLCBBSUMpCiAgcHJpbnQocmVzdWx0cykKICBsYXlvdXQobWF0cml4KGMoMSwyLDMsNCksMiwyKSkgCiAgcGxvdChtb2RlbCkKICByZXR1cm4ocHJlZGljdChtb2RlbCwgZGYsIHNlID0gVFJVRSkpCn0KCiMgV2UnbGwgdXNlIHRoaXMgaGVscGVyIGZ1bmN0aW9uIHRvIGNvbXBhcmUgdHdvIG1vZGVscyAKY29tcGFyZV9tb2RlbHMgPC0gZnVuY3Rpb24obW9kZWxfYm90aCwgbW9kZWxfc2luZ2xlKXsKICBsYXlvdXQobWF0cml4KGMoMSwyLDMsNCksMiwyKSkgCiAgcGxvdChtb2RlbF9ib3RoKQogIEFJQyA9IEFJQyhtb2RlbF9ib3RoKQogIGFub3ZhcCA9IGFub3ZhKG1vZGVsX2JvdGgsIG1vZGVsX3NpbmdsZSkkYFByKD5GKWBbLTFdCiAgcmVzdWx0cyA8LSBzcHJpbnRmKCJBSUM6ICVmLCBBTk9WQSBwOiAlZiIsIEFJQywgYW5vdmFwKQogIHByaW50KHJlc3VsdHMpCiAgdmlmKG1vZGVsX2JvdGgpCn0KCmBgYAoKIyBSYXcgQWNjdXJhY3kKCiMjIEluaXRpYWwgU2NhdHRlcnMKCmBgYHtyLCBmaWcud2lkdGggPSAxMiwgZmlnLmhlaWdodD00fQoKU0lHTU9SUEhPTiA8LSBhbGxfZGF0YSAlPiUgCiAgc2VsZWN0KGMoTW9kZWwsIEZhbWlseSwgdHJhaW5fc2l6ZT1TSUdNT1JQSE9OX3RyYWluX3NpemUsIHRyYWluX2xlbW1hcz1TSUdNT1JQSE9OX3RyYWluX2xlbW1hcywgdGVzdF9hY2MgPSBTSUdNT1JQSE9OX2FjYykpClNJR01PUlBIT04kRGF0YSA8LSAiU0lHTU9SUEhPTiIgCgpHb2xkbWFuIDwtIGFsbF9kYXRhICU+JQogIHNlbGVjdChjKE1vZGVsLCBGYW1pbHksIHRyYWluX3NpemU9R29sZG1hbl90cmFpbl9zaXplLCB0cmFpbl9sZW1tYXM9R29sZG1hbl90cmFpbl9sZW1tYXMsIHRlc3RfYWNjID0gR29sZG1hbl9hY2MpKQpHb2xkbWFuJERhdGEgPC0gIkdvbGRtYW4gZXQgYWwuIgoKcmJpbmQoU0lHTU9SUEhPTiwgR29sZG1hbikgJT4lCiAgZ2dwbG90KGFlcyhsb2codHJhaW5fc2l6ZSksIGxvZyh0ZXN0X2FjYyksIGNvbG9yPURhdGEpKSArIAogIGZhY2V0X2dyaWQofmZhY3RvcihNb2RlbCwgbGV2ZWxzPWMoIkNIUi1UUk0iLCAiRGVlcFNwaW4iLCAiQ0xVWkgiKSkpKwogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygidHVycXVvaXNlIiwgInB1cnBsZSIpKSArCiAgZ2VvbV9wb2ludChhZXMoc2hhcGU9RmFtaWx5KSwgc2l6ZT0zLCBhbHBoYT0wLjYpICsgCiAgdGhlbWVfYncoKSArIAogIHhsYWIoIlRyYWluaW5nIFNpemUsIExvZyBTY2FsZSIpICsgCiAgeWxhYigiVGVzdCBBY2N1cmFjeSwgTG9nIFNjYWxlIikgKyAKICBnZ3RpdGxlKCJUZXN0IEFjY3VyYWN5IHZzLiBUcmFpbmluZyBTaXplIikgKyAKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0PTAuNSwgc2l6ZT0xNikpIAoKZ2dzYXZlKCJmaWd1cmVzL3RyYWluX3NpemVfYWNjdXJhY3kucGRmIiwgZGV2aWNlPSJwZGYiLCB3aWR0aCA9IDEyLCBoZWlnaHQ9NCkKCmBgYAoKYGBge3IsIGZpZy53aWR0aCA9IDEyLCBmaWcuaGVpZ2h0PTR9CnJiaW5kKFNJR01PUlBIT04sIEdvbGRtYW4pICU+JQogIGdncGxvdChhZXMobG9nKHRyYWluX2xlbW1hcyksIGxvZyh0ZXN0X2FjYyksIGNvbG9yPURhdGEpKSArIAogIGZhY2V0X2dyaWQofmZhY3RvcihNb2RlbCwgbGV2ZWxzPWMoIkNIUi1UUk0iLCAiRGVlcFNwaW4iLCAiQ0xVWkgiKSkpKwogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygidHVycXVvaXNlIiwgInB1cnBsZSIpKSArCiAgZ2VvbV9wb2ludChhZXMoc2hhcGU9RmFtaWx5KSwgc2l6ZT0zLCBhbHBoYT0wLjYpICsgCiAgdGhlbWVfYncoKSArIAogIHhsYWIoIlRyYWluaW5nIExlbW1hcywgTG9nIFNjYWxlIikgKyAKICB5bGFiKCJUZXN0IEFjY3VyYWN5LCBMb2cgU2NhbGUiKSArIAogIGdndGl0bGUoIlRlc3QgQWNjdXJhY3kgdnMuIFRyYWluaW5nIExlbW1hcyIpICsgCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdD0wLjUsIHNpemU9MTYpKSAKCmdnc2F2ZSgiZmlndXJlcy90cmFpbl9sZW1tYXNfYWNjdXJhY3kucGRmIiwgZGV2aWNlPSJwZGYiLCB3aWR0aCA9IDEyLCBoZWlnaHQ9NCkKYGBgCgojIyBCYXNpYyBDb3JyZWxhdGlvbmFsIFN0YXRzCgpgYGB7cn0KY29ycmVsYXRpb25zKGxvZyhDSFJUUk0kR29sZG1hbl90cmFpbl9zaXplKSwgbG9nKENIUlRSTSRHb2xkbWFuX2FjYyArIDEpKQpzcHJpbnRmKCIiKQpjb3JyZWxhdGlvbnMobG9nKERlZXBTcGluJEdvbGRtYW5fdHJhaW5fc2l6ZSksIGxvZyhEZWVwU3BpbiRHb2xkbWFuX2FjYyArIDEpKQpzcHJpbnRmKCIiKQpjb3JyZWxhdGlvbnMobG9nKENMVVpIJEdvbGRtYW5fdHJhaW5fc2l6ZSksIGxvZyhDTFVaSCRHb2xkbWFuX2FjYyArIDEpKQpgYGAKCmBgYHtyfQpjb3JyZWxhdGlvbnMobG9nKENIUlRSTSRTSUdNT1JQSE9OX3RyYWluX3NpemUpLCBsb2coQ0hSVFJNJFNJR01PUlBIT05fYWNjKzEpKQpzcHJpbnRmKCIiKQpjb3JyZWxhdGlvbnMobG9nKERlZXBTcGluJFNJR01PUlBIT05fdHJhaW5fc2l6ZSksIGxvZyhEZWVwU3BpbiRTSUdNT1JQSE9OX2FjYysxKSkKc3ByaW50ZigiIikKY29ycmVsYXRpb25zKGxvZyhDTFVaSCRTSUdNT1JQSE9OX3RyYWluX3NpemUpLCBsb2coQ0xVWkgkU0lHTU9SUEhPTl9hY2MrMSkpCmBgYAoKYGBge3J9CmNvcnJlbGF0aW9ucyhsb2coQ0hSVFJNJEdvbGRtYW5fdHJhaW5fbGVtbWFzKSwgbG9nKENIUlRSTSRHb2xkbWFuX2FjYysxKSkKc3ByaW50ZigiIikKY29ycmVsYXRpb25zKGxvZyhEZWVwU3BpbiRHb2xkbWFuX3RyYWluX2xlbW1hcyksIGxvZyhEZWVwU3BpbiRHb2xkbWFuX2FjYysxKSkKc3ByaW50ZigiIikKY29ycmVsYXRpb25zKGxvZyhDTFVaSCRHb2xkbWFuX3RyYWluX2xlbW1hcyksIGxvZyhDTFVaSCRHb2xkbWFuX2FjYysxKSkKYGBgCgpgYGB7cn0KY29ycmVsYXRpb25zKGxvZyhDSFJUUk0kU0lHTU9SUEhPTl90cmFpbl9sZW1tYXMpLCBsb2coQ0hSVFJNJFNJR01PUlBIT05fYWNjKzEpKQpzcHJpbnRmKCIiKQpjb3JyZWxhdGlvbnMobG9nKERlZXBTcGluJFNJR01PUlBIT05fdHJhaW5fbGVtbWFzKSwgbG9nKERlZXBTcGluJFNJR01PUlBIT05fYWNjKzEpKQpzcHJpbnRmKCIiKQpjb3JyZWxhdGlvbnMobG9nKENMVVpIJFNJR01PUlBIT05fdHJhaW5fbGVtbWFzKSwgbG9nKENMVVpIJFNJR01PUlBIT05fYWNjKzEpKQpgYGAKCiMjIEdMTWVyIEF0dGVtcHQKCmBgYHtyfQpkYXRhIDwtIHJiaW5kKFNJR01PUlBIT04sIEdvbGRtYW4pCmRhdGEkc2NhbGVkX2FjYyA8LSAoZGF0YSR0ZXN0X2FjYy8xMDAgKiAyMDMgKyAuNSkgLyAyMDQKCgptb2RlbCA8LSBnbG1lcihzY2FsZWRfYWNjIH4gbG9nKHRyYWluX2xlbW1hcykgKyBsb2codHJhaW5fc2l6ZSkgKyAoMXxNb2RlbCkgICsgKCB0cmFpbl9zaXplICsgdHJhaW5fbGVtbWFzIHxGYW1pbHkpICsgKHRyYWluX2xlbW1hcyB8RGF0YSksIGRhdGE9ZGF0YSwgZmFtaWx5PWJpbm9taWFsKQpzdW1tYXJ5KG1vZGVsKQpgYGAKCiMjIEJldGEgUmVncmVzc2lvbgoKYGBge3J9CnJiaW5kKENMVVpILCBDSFJUUk0sIERlZXBTcGluKSAlPiUKICBnZ3Bsb3QoYWVzKHggPSBHb2xkbWFuX3NjYWxlZF9hY2MsIGNvbG9yPU1vZGVsKSkgKyAKICBnZW9tX2RlbnNpdHkoYWVzKGZpbGw9IE1vZGVsKSwgYWxwaGEgPSAwLjUpICsgCiAgc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcz1jKCJwdXJwbGUiLCAiZ29sZCIsICJ0dXJxdW9pc2UiKSkgKwogIHNjYWxlX2ZpbGxfbWFudWFsKHZhbHVlcz1jKCJwdXJwbGUiLCAiZ29sZCIsICJ0dXJxdW9pc2UiKSkgKwogIHRoZW1lX2J3KCkgKyAKICB5bGFiKCJEZW5zaXR5IikgKyAKICB4bGFiKCJHb2xkbWFuIGV0IGFsLiBUZXN0IEFjY3VyYWN5IikgKyAKICBnZ3RpdGxlKCJEZW5zaXR5IG9mIEdvbGRtYW4gZXQgYWwuIFRlc3QgQWNjdXJhY3kiKSArIAogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3Q9MC41LCBzaXplPTE2KSkgCgpnZ3NhdmUoImZpZ3VyZXMvYWNjdXJhY3lfZGlzdHJpYnV0aW9uLnBkZiIsIGRldmljZT0icGRmIiwgd2lkdGggPSA4LCBoZWlnaHQ9NSkKYGBgCgpgYGB7cn0KQ0hSVFJNJEdvbGRtYW5fc2NhbGVkX2FjYyA8LSAoQ0hSVFJNJEdvbGRtYW5fYWNjLzEwMCAqIDMzICsgLjUpIC8gMzQKCnNpemVfbW9kZWwgPSBnYW0oR29sZG1hbl9zY2FsZWRfYWNjIH4gbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksIGZhbWlseT1iZXRhcihsaW5rPSJsb2dpdCIpLCBkYXRhID0gQ0hSVFJNKQpzdW1tYXJ5KHNpemVfbW9kZWwpCnJlcyA8LSBwcmVkaWN0KHNpemVfbW9kZWwsIG5ld2RhdGEgPSBDSFJUUk0sIHNlLmZpdCA9IFRSVUUsIHR5cGUgPSAicmVzcG9uc2UiKQpDSFJUUk0kc2l6ZV9maXQgPC0gcmVzJGZpdApDSFJUUk0kc2l6ZV9zZSA8LSByZXMkc2UuZml0CgoKbGVtbWFfbW9kZWwgPSBnYW0oR29sZG1hbl9zY2FsZWRfYWNjIH4gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSwgZmFtaWx5PWJldGFyKGxpbms9ImxvZ2l0IiksIGRhdGEgPSBDSFJUUk0pCnJlcyA8LSBwcmVkaWN0KGxlbW1hX21vZGVsLCBuZXdkYXRhID0gQ0hSVFJNLCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKQ0hSVFJNJGxlbW1hX2ZpdCA8LSByZXMkZml0CkNIUlRSTSRsZW1tYV9zZSA8LSByZXMkc2UuZml0CnN1bW1hcnkobGVtbWFfbW9kZWwpCgpib3RoX21vZGVsIDwtIGdhbShHb2xkbWFuX3NjYWxlZF9hY2MgfiBsb2coR29sZG1hbl90cmFpbl9sZW1tYXMpICsgbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksIGZhbWlseT1iZXRhcihsaW5rPSJsb2dpdCIpLCBkYXRhID0gQ0hSVFJNKQpzdW1tYXJ5KGJvdGhfbW9kZWwpCgpBSUMoc2l6ZV9tb2RlbCkKQUlDKGxlbW1hX21vZGVsKQpBSUMoYm90aF9tb2RlbCkKCgpCSUMoc2l6ZV9tb2RlbCkKQklDKGxlbW1hX21vZGVsKQpCSUMoYm90aF9tb2RlbCkKCnNpemVfbW9kZWwkY29lZmZpY2llbnRzCmxlbW1hX21vZGVsJGNvZWZmaWNpZW50cwpgYGAKCmBgYHtyfQpEZWVwU3BpbiRHb2xkbWFuX3NjYWxlZF9hY2MgPC0gKERlZXBTcGluJEdvbGRtYW5fYWNjLzEwMCAqIDMzICsgLjUpIC8gMzQKCnNpemVfbW9kZWwgPSBnYW0oR29sZG1hbl9zY2FsZWRfYWNjIH4gbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksIGZhbWlseT1iZXRhcihsaW5rPSJsb2dpdCIpLCBkYXRhID0gRGVlcFNwaW4pCnJlcyA8LSBwcmVkaWN0KHNpemVfbW9kZWwsIG5ld2RhdGEgPSBEZWVwU3Bpbiwgc2UuZml0ID0gVFJVRSwgdHlwZSA9ICJyZXNwb25zZSIpCnN1bW1hcnkoc2l6ZV9tb2RlbCkKRGVlcFNwaW4kc2l6ZV9maXQgPC0gcmVzJGZpdApEZWVwU3BpbiRzaXplX3NlIDwtIHJlcyRzZS5maXQKCmxlbW1hX21vZGVsID0gZ2FtKEdvbGRtYW5fc2NhbGVkX2FjYyB+IGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcyksIGZhbWlseT1iZXRhcihsaW5rPSJsb2dpdCIpLCBkYXRhID0gRGVlcFNwaW4pCnJlcyA8LSBwcmVkaWN0KGxlbW1hX21vZGVsLCBuZXdkYXRhID0gRGVlcFNwaW4sIHNlLmZpdCA9IFRSVUUsIHR5cGUgPSAicmVzcG9uc2UiKQpzdW1tYXJ5KGxlbW1hX21vZGVsKQpEZWVwU3BpbiRsZW1tYV9maXQgPC0gcmVzJGZpdApEZWVwU3BpbiRsZW1tYV9zZSA8LSByZXMkc2UuZml0Cgpib3RoX21vZGVsIDwtIGdhbShHb2xkbWFuX3NjYWxlZF9hY2MgfiBsb2coR29sZG1hbl90cmFpbl9zaXplKSArIGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcyksIGZhbWlseT1iZXRhcihsaW5rPSJsb2dpdCIpLCBkYXRhID0gRGVlcFNwaW4pCnN1bW1hcnkoYm90aF9tb2RlbCkKCkFJQyhzaXplX21vZGVsKQpBSUMobGVtbWFfbW9kZWwpCkFJQyhib3RoX21vZGVsKQoKCkJJQyhzaXplX21vZGVsKQpCSUMobGVtbWFfbW9kZWwpCkJJQyhib3RoX21vZGVsKQoKc2l6ZV9tb2RlbCRjb2VmZmljaWVudHMKbGVtbWFfbW9kZWwkY29lZmZpY2llbnRzCmBgYAoKYGBge3J9CkNMVVpIJEdvbGRtYW5fc2NhbGVkX2FjYyA8LSAoQ0xVWkgkR29sZG1hbl9hY2MvMTAwICogMzMgKyAuNSkgLyAzNAoKc2l6ZV9tb2RlbCA9IGdhbShHb2xkbWFuX3NjYWxlZF9hY2MgfiBsb2coR29sZG1hbl90cmFpbl9zaXplKSwgZmFtaWx5PWJldGFyKGxpbms9ImxvZ2l0IiksIGRhdGEgPSBDTFVaSCkKcmVzIDwtIHByZWRpY3Qoc2l6ZV9tb2RlbCwgbmV3ZGF0YSA9IENMVVpILCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKQ0xVWkgkc2l6ZV9maXQgPC0gcmVzJGZpdApDTFVaSCRzaXplX3NlIDwtIHJlcyRzZS5maXQKc3VtbWFyeShzaXplX21vZGVsKQoKbGVtbWFfbW9kZWwgPSBnYW0oR29sZG1hbl9zY2FsZWRfYWNjIH4gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSwgZmFtaWx5PWJldGFyKGxpbms9ImxvZ2l0IiksIGRhdGEgPSBDTFVaSCkKcmVzIDwtIHByZWRpY3QobGVtbWFfbW9kZWwsIG5ld2RhdGEgPSBDTFVaSCwgc2UuZml0ID0gVFJVRSwgdHlwZSA9ICJyZXNwb25zZSIpCkNMVVpIJGxlbW1hX2ZpdCA8LSByZXMkZml0CkNMVVpIJGxlbW1hX3NlIDwtIHJlcyRzZS5maXQKc3VtbWFyeShsZW1tYV9tb2RlbCkKCmJvdGhfbW9kZWwgPC0gZ2FtKEdvbGRtYW5fc2NhbGVkX2FjYyB+IGxvZyhHb2xkbWFuX3RyYWluX3NpemUpICsgbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSwgZmFtaWx5PWJldGFyKGxpbms9ImxvZ2l0IiksIGRhdGEgPSBDTFVaSCkKc3VtbWFyeShib3RoX21vZGVsKQoKCkFJQyhzaXplX21vZGVsKQpBSUMobGVtbWFfbW9kZWwpCkFJQyhib3RoX21vZGVsKQoKCkJJQyhzaXplX21vZGVsKQpCSUMobGVtbWFfbW9kZWwpCkJJQyhib3RoX21vZGVsKQoKc2l6ZV9tb2RlbCRjb2VmZmljaWVudHMKbGVtbWFfbW9kZWwkY29lZmZpY2llbnRzCmBgYAoKYGBge3IsIGZpZy53aWR0aCA9IDEyLCBmaWcuaGVpZ2h0PTR9CnJiaW5kKENMVVpILCBDSFJUUk0sIERlZXBTcGluKSAlPiUgCiAgZ2dwbG90KGFlcyhHb2xkbWFuX3RyYWluX3NpemUsIEdvbGRtYW5fc2NhbGVkX2FjYykpICsgCiAgZ2VvbV9wb2ludChhZXMoY29sb3I9RmFtaWx5KSwgc2l6ZSA9IDMsIGFscGhhID0gMC42KSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygidHVycXVvaXNlIiwgInB1cnBsZSIsICJnb2xkIikpICsgCiAgZmFjZXRfZ3JpZCh+ZmFjdG9yKE1vZGVsLCBsZXZlbHM9YygiQ0hSLVRSTSIsICJEZWVwU3BpbiIsICJDTFVaSCIpKSkgKyAKICBnZW9tX2xpbmUoYWVzKEdvbGRtYW5fdHJhaW5fc2l6ZSwgc2l6ZV9maXQpKSArIAogIGdlb21fcmliYm9uKGFlcyh4ID0gR29sZG1hbl90cmFpbl9zaXplLCAKICAgICAgICAgICAgICAgICAgeW1pbiA9IHNpemVfZml0IC0gMiAqIHNpemVfc2UsCiAgICAgICAgICAgICAgICAgIHltYXggPSBzaXplX2ZpdCArIDIgKiBzaXplX3NlLAogICAgICAgICAgICAgICAgICApLCBmaWxsID0iZ3JleSIsCiAgICAgICAgICAgICAgICAgIGFscGhhID0gMC40KSArIAogIHRoZW1lX2J3KCkgKyAKICB4bGFiKCJHb2xkbWFuIFRyYWluaW5nIFNpemUiKSArIAogIHlsYWIoIkFjY3VyYWN5IikgKyAKICBnZ3RpdGxlKCJUcmFpbmluZyBTaXplIHZzLiBBY2N1cmFjeSwgR29sZG1hbiBldCBhbC4gRGF0YSIpICsgCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdD0wLjUsIHNpemU9MTYpKSAKCmdnc2F2ZSgiZmlndXJlcy90cmFpbl9zaXplX2JldGEucGRmIiwgZGV2aWNlPSJwZGYiLCB3aWR0aCA9IDEyLCBoZWlnaHQ9NCkKYGBgCgpgYGB7ciwgZmlnLndpZHRoID0gMTIsIGZpZy5oZWlnaHQ9NH0KcmJpbmQoQ0xVWkgsIENIUlRSTSwgRGVlcFNwaW4pICU+JSAKICBnZ3Bsb3QoYWVzKEdvbGRtYW5fdHJhaW5fbGVtbWFzLCBHb2xkbWFuX3NjYWxlZF9hY2MpKSArIAogIGdlb21fcG9pbnQoYWVzKGNvbG9yPUZhbWlseSksIHNpemUgPSAzLCBhbHBoYSA9IDAuNikgKyAKICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzPWMoInR1cnF1b2lzZSIsICJwdXJwbGUiLCAiZ29sZCIpKSArIAogIGZhY2V0X2dyaWQofmZhY3RvcihNb2RlbCwgbGV2ZWxzPWMoIkNIUi1UUk0iLCAiRGVlcFNwaW4iLCAiQ0xVWkgiKSkpICsgCiAgZ2VvbV9saW5lKGFlcyhHb2xkbWFuX3RyYWluX2xlbW1hcywgbGVtbWFfZml0KSkgKyAKICBnZW9tX3JpYmJvbihhZXMoeCA9IEdvbGRtYW5fdHJhaW5fbGVtbWFzLCAKICAgICAgICAgICAgICAgICAgeW1pbiA9IGxlbW1hX2ZpdCAtIDIgKiBsZW1tYV9zZSwKICAgICAgICAgICAgICAgICAgeW1heCA9IGxlbW1hX2ZpdCArIDIgKiBsZW1tYV9zZSwKICAgICAgICAgICAgICAgICAgKSwgZmlsbCA9ImdyZXkiLAogICAgICAgICAgICAgICAgICBhbHBoYSA9IDAuNCkgKyAKICB0aGVtZV9idygpICsgCiAgeGxhYigiR29sZG1hbiBUcmFpbmluZyBMZW1tYXMiKSArIAogIHlsYWIoIkFjY3VyYWN5IikgKyAKICBnZ3RpdGxlKCJUcmFpbmluZyBMZW1tYXMgdnMuIEFjY3VyYWN5LCBHb2xkbWFuIGV0IGFsLiBEYXRhIikgKyAKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0PTAuNSwgc2l6ZT0xNikpIAoKI2dnc2F2ZSgiZmlndXJlcy90cmFpbl9sZW1tYXNfYmV0YS5wZGYiLCBkZXZpY2U9InBkZiIsIHdpZHRoID0gMTIsIGhlaWdodD00KQpgYGAKCiMgQWNjdXJhY3kgRHJvcAoKIyMgSW5pdGlhbCBWaXN1YWxpemF0aW9uCgojIyMgUmF3IFBsb3RzCgpgYGB7ciwgZmlnLndpZHRoID0gOSwgZmlnLmhlaWdodD0zfQphbGxfZGF0YSAlPiUgCiAgZ2dwbG90KGFlcyh4ID0gR29sZG1hbl90cmFpbl9zaXplLCB5ID0gYWNjX2Ryb3ApKSArIAogIGZhY2V0X2dyaWQofmZhY3RvcihNb2RlbCwgbGV2ZWxzPWMoIkNIUi1UUk0iLCAiRGVlcFNwaW4iLCAiQ0xVWkgiKSkpICsKICBnZW9tX3BvaW50KGFlcyhjb2xvcj1GYW1pbHkpLCBzaXplPTMsIGFscGhhID0gMC42KSArIAogIHRoZW1lX2J3KCkgKyAKICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzPWMoInB1cnBsZSIsInR1cnF1b2lzZSIsICJnb2xkIikpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0PTAuNSwgc2l6ZT0xNikpICsKICB4bGFiKCJHb2xkbWFuIGV0IGFsLiBUcmFpbmluZyBTaXplIikgKyAKICB5bGFiKCJBY2N1cmFjeSBEcm9wIikgKyAKICBnZ3RpdGxlKCJUcmFpbmluZyBTaXplIHZzLiBBY2N1cmFjeSBEcm9wIikgCmBgYAoKYGBge3IsIGZpZy53aWR0aCA9IDksIGZpZy5oZWlnaHQ9M30KYWxsX2RhdGEgJT4lIAogIGdncGxvdChhZXMoeCA9IEdvbGRtYW5fdHJhaW5fbGVtbWFzLCB5ID0gYWNjX2Ryb3ApKSArIAogIGZhY2V0X2dyaWQofmZhY3RvcihNb2RlbCwgbGV2ZWxzPWMoIkNIUi1UUk0iLCAiRGVlcFNwaW4iLCAiQ0xVWkgiKSkpICsKICBnZW9tX3BvaW50KGFlcyhjb2xvcj1GYW1pbHkpLCBzaXplPTMsIGFscGhhID0gMC42KSArIAogIHRoZW1lX2J3KCkgKyAKICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzPWMoInB1cnBsZSIsInR1cnF1b2lzZSIsICJnb2xkIikpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0PTAuNSwgc2l6ZT0xNikpICsKICB4bGFiKCJHb2xkbWFuIGV0IGFsLiBUcmFpbmluZyBMZW1tYXMiKSArIAogIHlsYWIoIkFjY3VyYWN5IERyb3AiKSArIAogIGdndGl0bGUoIlRyYWluaW5nIExlbW1hcyB2cy4gQWNjdXJhY3kgRHJvcCIpCmBgYAoKYGBge3IsIGZpZy53aWR0aCA9IDksIGZpZy5oZWlnaHQ9M30KYWxsX2RhdGEgJT4lIAogIGdncGxvdChhZXMoeCA9IHRyYWluX2xlbW1hX2RpZmZfcmF3LCB5ID0gYWNjX2Ryb3ApKSArIAogIGZhY2V0X2dyaWQofmZhY3RvcihNb2RlbCwgbGV2ZWxzPWMoIkNIUi1UUk0iLCAiRGVlcFNwaW4iLCAiQ0xVWkgiKSkpICsKICBnZW9tX3BvaW50KGFlcyhjb2xvcj1GYW1pbHkpLCBzaXplPTMsIGFscGhhID0gMC42KSArIAogIHRoZW1lX2J3KCkgKyAKICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzPWMoInB1cnBsZSIsInR1cnF1b2lzZSIsICJnb2xkIikpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0PTAuNSwgc2l6ZT0xNikpICsKICB4bGFiKCJEcm9wIGluIHRoZSBOdW1iZXIgb2YgVHJhaW5pbmcgTGVtbWFzIikgKyAKICB5bGFiKCJBY2N1cmFjeSBEcm9wIikgKyAKICBnZ3RpdGxlKCJUcmFpbmluZyBMZW1tYSBEcm9wIHZzLiBBY2N1cmFjeSBEcm9wIikKYGBgCgojIyMgTG9nLUxvZyBQbG90cyBhbmQgTE0gRml0IExpbmVzCgpgYGB7ciwgZmlnLndpZHRoID0gOSwgZmlnLmhlaWdodD0zfQphbGxfZGF0YSAlPiUgCiAgZ2dwbG90KGFlcyh4ID0gbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksIHkgPSBsb2dfYWNjX2Ryb3ApKSArIAogIGZhY2V0X2dyaWQofmZhY3RvcihNb2RlbCwgbGV2ZWxzPWMoIkNIUi1UUk0iLCAiRGVlcFNwaW4iLCAiQ0xVWkgiKSkpICsKICBnZW9tX3BvaW50KGFlcyhjb2xvcj1GYW1pbHkpLCBzaXplPTMsIGFscGhhID0gMC42KSArIAogIHRoZW1lX2J3KCkgKyAKICBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzPWMoInB1cnBsZSIsInR1cnF1b2lzZSIsICJnb2xkIikpICsKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0PTAuNSwgc2l6ZT0xNikpICsKICB4bGFiKCJHb2xkbWFuIGV0IGFsLiBUcmFpbmluZyBTaXplLCBMb2cgU2NhbGUiKSArIAogIHlsYWIoIkFjY3VyYWN5IERyb3AsIExvZyBTY2FsZSIpICsgCiAgZ2d0aXRsZSgiVHJhaW5pbmcgU2l6ZSB2cy4gQWNjdXJhY3kgRHJvcCIpICsgCiAgc3RhdF9zbW9vdGgobWV0aG9kPSJsbSIsIGNvbG9yPSJncmV5IikKYGBgCgpgYGB7ciwgZmlnLndpZHRoID0gOSwgZmlnLmhlaWdodD0zfQphbGxfZGF0YSAlPiUgCiAgZ2dwbG90KGFlcyh4ID0gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSwgeSA9IGxvZ19hY2NfZHJvcCkpICsgCiAgZmFjZXRfZ3JpZCh+ZmFjdG9yKE1vZGVsLCBsZXZlbHM9YygiQ0hSLVRSTSIsICJEZWVwU3BpbiIsICJDTFVaSCIpKSkgKwogIGdlb21fcG9pbnQoYWVzKGNvbG9yPUZhbWlseSksIHNpemU9MywgYWxwaGEgPSAwLjYpICsgCiAgdGhlbWVfYncoKSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygicHVycGxlIiwidHVycXVvaXNlIiwgImdvbGQiKSkgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3Q9MC41LCBzaXplPTE2KSkgKwogIHhsYWIoIkdvbGRtYW4gZXQgYWwuIFRyYWluaW5nIExlbW1hcywgTG9nIFNjYWxlIikgKyAKICB5bGFiKCJBY2N1cmFjeSBEcm9wLCBMb2cgU2NhbGUiKSArIAogIGdndGl0bGUoIlRyYWluaW5nIExlbW1hcyB2cy4gQWNjdXJhY3kgRHJvcCIpICsgCiAgc3RhdF9zbW9vdGgobWV0aG9kPSJsbSIsIGNvbG9yPSJncmV5IikKYGBgCgpgYGB7ciwgZmlnLndpZHRoID0gOSwgZmlnLmhlaWdodD0zfQphbGxfZGF0YSAlPiUgCiAgZ2dwbG90KGFlcyh4ID0gbG9nKC0xKnRyYWluX2xlbW1hX2RpZmZfcmF3KSwgeSA9IGxvZ19hY2NfZHJvcCkpICsgCiAgZmFjZXRfZ3JpZCh+ZmFjdG9yKE1vZGVsLCBsZXZlbHM9YygiQ0hSLVRSTSIsICJEZWVwU3BpbiIsICJDTFVaSCIpKSkgKwogIGdlb21fcG9pbnQoYWVzKGNvbG9yPUZhbWlseSksIHNpemU9MywgYWxwaGEgPSAwLjYpICsgCiAgdGhlbWVfYncoKSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygicHVycGxlIiwidHVycXVvaXNlIiwgImdvbGQiKSkgKwogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3Q9MC41LCBzaXplPTE2KSkgKwogIHhsYWIoIkRyb3AgaW4gVHJhaW5pbmcgTGVtbWFzLCBMb2cgU2NhbGUiKSArIAogIHlsYWIoIkFjY3VyYWN5IERyb3AsIExvZyBTY2FsZSIpICsgCiAgZ2d0aXRsZSgiTGVtbWEgRHJvcCB2cy4gQWNjdXJhY3kgRHJvcCIpICsgCiAgc3RhdF9zbW9vdGgobWV0aG9kPSJsbSIsIGNvbG9yPSJncmV5IikKYGBgCgojIyBCYXNpYyBDb3JyZWxhdGlvbmFsIFN0YXRzCgpgYGB7cn0KQ0hSVFJNJHNjYWxlZF9kcm9wIDwtIC0xKihDSFJUUk0kR29sZG1hbl9hY2MgLSBDSFJUUk0kU0lHTU9SUEhPTl9hY2MgLSAyKS8xMDAKRGVlcFNwaW4kc2NhbGVkX2Ryb3AgPC0gLTEqKERlZXBTcGluJEdvbGRtYW5fYWNjIC0gRGVlcFNwaW4kU0lHTU9SUEhPTl9hY2MgLSAyKS8xMDAKQ0xVWkgkc2NhbGVkX2Ryb3AgPC0gLTEqKENMVVpIJEdvbGRtYW5fYWNjIC0gQ0xVWkgkU0lHTU9SUEhPTl9hY2MgLSA0KS8xMDAKCmNvcnJlbGF0aW9ucyhsb2coQ0hSVFJNJEdvbGRtYW5fdHJhaW5fc2l6ZSksIGxvZyhDSFJUUk0kc2NhbGVkX2Ryb3ApKQpzcHJpbnRmKCIiKQpjb3JyZWxhdGlvbnMobG9nKERlZXBTcGluJEdvbGRtYW5fdHJhaW5fc2l6ZSksIGxvZyhEZWVwU3BpbiRzY2FsZWRfZHJvcCkpCnNwcmludGYoIiIpCmNvcnJlbGF0aW9ucyhsb2coQ0xVWkgkR29sZG1hbl90cmFpbl9zaXplKSwgbG9nKENMVVpIJHNjYWxlZF9kcm9wKSkKYGBgCgpgYGB7cn0KY29ycmVsYXRpb25zKGxvZyhDSFJUUk0kR29sZG1hbl90cmFpbl9sZW1tYXMpLCBsb2coQ0hSVFJNJHNjYWxlZF9kcm9wKSkKc3ByaW50ZigiIikKY29ycmVsYXRpb25zKGxvZyhEZWVwU3BpbiRHb2xkbWFuX3RyYWluX2xlbW1hcyksIGxvZyhEZWVwU3BpbiRzY2FsZWRfZHJvcCkpCnNwcmludGYoIiIpCmNvcnJlbGF0aW9ucyhsb2coQ0xVWkgkR29sZG1hbl90cmFpbl9sZW1tYXMpLCBsb2coQ0xVWkgkc2NhbGVkX2Ryb3ApKQpgYGAKCmBgYHtyfQpjb3JyZWxhdGlvbnMobG9nKC0xKkNIUlRSTSR0cmFpbl9sZW1tYV9kaWZmX3JhdyksIGxvZyhDSFJUUk0kc2NhbGVkX2Ryb3ApKQpzcHJpbnRmKCIiKQpjb3JyZWxhdGlvbnMobG9nKC0xKkRlZXBTcGluJHRyYWluX2xlbW1hX2RpZmZfcmF3KSwgbG9nKERlZXBTcGluJHNjYWxlZF9kcm9wKSkKc3ByaW50ZigiIikKY29ycmVsYXRpb25zKGxvZygtMSpDTFVaSCR0cmFpbl9sZW1tYV9kaWZmX3JhdyksIGxvZyhDTFVaSCRzY2FsZWRfZHJvcCkpCmBgYAoKIyMgQmV0YSBSZWdyZXNzaW9uCgpgYGB7cn0KcmJpbmQoQ0xVWkgsIENIUlRSTSwgRGVlcFNwaW4pICU+JQogIGdncGxvdChhZXMoeCA9IHNjYWxlZF9kcm9wLCBjb2xvcj1Nb2RlbCkpICsgCiAgZ2VvbV9kZW5zaXR5KGFlcyhmaWxsPSBNb2RlbCksIGFscGhhID0gMC41KSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygicHVycGxlIiwgImdvbGQiLCAidHVycXVvaXNlIikpICsKICBzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXM9YygicHVycGxlIiwgImdvbGQiLCAidHVycXVvaXNlIikpICsKICB0aGVtZV9idygpICsgCiAgeWxhYigiRGVuc2l0eSIpICsgCiAgeGxhYigiQWNjdXJhY3kgRHJvcCIpICsgCiAgZ2d0aXRsZSgiRGVuc2l0eSBvZiBBY2N1cmFjeSBEcm9wIikgKyAKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0PTAuNSwgc2l6ZT0xNikpIAoKZ2dzYXZlKCJmaWd1cmVzL2Ryb3BfZGlzdHJpYnV0aW9uLnBkZiIsIGRldmljZT0icGRmIiwgd2lkdGggPSA4LCBoZWlnaHQ9NSkKYGBgCgpgYGB7cn0Kc2l6ZV9tb2RlbCA9IGdhbShzY2FsZWRfZHJvcCB+IGxvZyhHb2xkbWFuX3RyYWluX3NpemUpLCBmYW1pbHk9YmV0YXIobGluaz0ibG9naXQiKSwgZGF0YSA9IENIUlRSTSkKCnN1bW1hcnkoc2l6ZV9tb2RlbCkKCnJlcyA8LSBwcmVkaWN0KHNpemVfbW9kZWwsIG5ld2RhdGEgPSBDSFJUUk0sIHNlLmZpdCA9IFRSVUUsIHR5cGU9InJlc3BvbnNlIikKQ0hSVFJNJHNpemVfZml0IDwtIHJlcyRmaXQKQ0hSVFJNJHNpemVfc2UgPC0gcmVzJHNlLmZpdAoKbGVtbWFfbW9kZWwgPSBnYW0oc2NhbGVkX2Ryb3AgfiBsb2coR29sZG1hbl90cmFpbl9sZW1tYXMpLCBmYW1pbHk9YmV0YXIobGluaz0ibG9naXQiKSwgZGF0YSA9IENIUlRSTSkKCnN1bW1hcnkobGVtbWFfbW9kZWwpCgpyZXMgPC0gcHJlZGljdChsZW1tYV9tb2RlbCwgbmV3ZGF0YSA9IENIUlRSTSwgc2UuZml0ID0gVFJVRSwgdHlwZT0icmVzcG9uc2UiKQpDSFJUUk0kbGVtbWFfZml0IDwtIHJlcyRmaXQKQ0hSVFJNJGxlbW1hX3NlIDwtIHJlcyRzZS5maXQKCmRyb3BfbW9kZWwgPSBnYW0oc2NhbGVkX2Ryb3AgfiBsb2coLTEqdHJhaW5fbGVtbWFfZGlmZl9yYXcpLCBmYW1pbHk9YmV0YXIobGluaz0ibG9naXQiKSwgZGF0YSA9IENIUlRSTSkKCnN1bW1hcnkoZHJvcF9tb2RlbCkKCnJlcyA8LSBwcmVkaWN0KGRyb3BfbW9kZWwsIG5ld2RhdGEgPSBDSFJUUk0sIHNlLmZpdCA9IFRSVUUsIHR5cGU9InJlc3BvbnNlIikKQ0hSVFJNJGRyb3BfZml0IDwtIHJlcyRmaXQKQ0hSVFJNJGRyb3Bfc2UgPC0gcmVzJHNlLmZpdAoKYm90aF9tb2RlbCA8LSBnYW0oc2NhbGVkX2Ryb3AgfiBsb2coR29sZG1hbl90cmFpbl9sZW1tYXMpICArIGxvZygtMSogdHJhaW5fbGVtbWFfZGlmZl9yYXcpICsgbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksIGZhbWlseT1iZXRhcihsaW5rPSJsb2dpdCIpLCBkYXRhID0gQ0hSVFJNKQoKc3VtbWFyeShib3RoX21vZGVsKQoKc2l6ZV9tb2RlbCRjb2VmZmljaWVudHMKbGVtbWFfbW9kZWwkY29lZmZpY2llbnRzCmRyb3BfbW9kZWwkY29lZmZpY2llbnRzCgpBSUMoc2l6ZV9tb2RlbCkKQUlDKGxlbW1hX21vZGVsKQpBSUMoZHJvcF9tb2RlbCkKCgpCSUMoc2l6ZV9tb2RlbCkKQklDKGxlbW1hX21vZGVsKQpCSUMoZHJvcF9tb2RlbCkKYGBgCgpgYGB7cn0Kc2l6ZV9tb2RlbCA9IGdhbShzY2FsZWRfZHJvcCB+IGxvZyhHb2xkbWFuX3RyYWluX3NpemUpLCBmYW1pbHk9YmV0YXIobGluaz0ibG9naXQiKSwgZGF0YSA9IERlZXBTcGluKQoKc3VtbWFyeShzaXplX21vZGVsKQoKcmVzIDwtIHByZWRpY3Qoc2l6ZV9tb2RlbCwgbmV3ZGF0YSA9IERlZXBTcGluLCBzZS5maXQgPSBUUlVFLCB0eXBlPSJyZXNwb25zZSIpCkRlZXBTcGluJHNpemVfZml0IDwtIHJlcyRmaXQKRGVlcFNwaW4kc2l6ZV9zZSA8LSByZXMkc2UuZml0CgpsZW1tYV9tb2RlbCA9IGdhbShzY2FsZWRfZHJvcCB+IGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcyksIGZhbWlseT1iZXRhcihsaW5rPSJsb2dpdCIpLCBkYXRhID0gRGVlcFNwaW4pCgpzdW1tYXJ5KGxlbW1hX21vZGVsKQoKcmVzIDwtIHByZWRpY3QobGVtbWFfbW9kZWwsIG5ld2RhdGEgPSBEZWVwU3Bpbiwgc2UuZml0ID0gVFJVRSwgdHlwZT0icmVzcG9uc2UiKQpEZWVwU3BpbiRsZW1tYV9maXQgPC0gcmVzJGZpdApEZWVwU3BpbiRsZW1tYV9zZSA8LSByZXMkc2UuZml0Cgpkcm9wX21vZGVsID0gZ2FtKHNjYWxlZF9kcm9wIH4gbG9nKC0xKnRyYWluX2xlbW1hX2RpZmZfcmF3KSwgZmFtaWx5PWJldGFyKGxpbms9ImxvZ2l0IiksIGRhdGEgPSBEZWVwU3BpbikKCnN1bW1hcnkoZHJvcF9tb2RlbCkKCnJlcyA8LSBwcmVkaWN0KGRyb3BfbW9kZWwsIG5ld2RhdGEgPSBEZWVwU3Bpbiwgc2UuZml0ID0gVFJVRSwgdHlwZT0icmVzcG9uc2UiKQpEZWVwU3BpbiRkcm9wX2ZpdCA8LSByZXMkZml0CkRlZXBTcGluJGRyb3Bfc2UgPC0gcmVzJHNlLmZpdAoKYm90aF9tb2RlbCA8LSBnYW0oc2NhbGVkX2Ryb3AgfiBsb2coLTEqdHJhaW5fbGVtbWFfZGlmZl9yYXcpICsgbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSkgKyBsb2coR29sZG1hbl90cmFpbl9sZW1tYXMpLCBmYW1pbHk9YmV0YXIobGluaz0ibG9naXQiKSwgZGF0YSA9IERlZXBTcGluKQoKc3VtbWFyeShib3RoX21vZGVsKQoKCnNpemVfbW9kZWwkY29lZmZpY2llbnRzCmxlbW1hX21vZGVsJGNvZWZmaWNpZW50cwpkcm9wX21vZGVsJGNvZWZmaWNpZW50cwoKQUlDKHNpemVfbW9kZWwpCkFJQyhsZW1tYV9tb2RlbCkKQUlDKGRyb3BfbW9kZWwpCgoKQklDKHNpemVfbW9kZWwpCkJJQyhsZW1tYV9tb2RlbCkKQklDKGRyb3BfbW9kZWwpCmBgYAoKYGBge3J9CnNpemVfbW9kZWwgPSBnYW0oc2NhbGVkX2Ryb3AgfiBsb2coR29sZG1hbl90cmFpbl9zaXplKSwgZmFtaWx5PWJldGFyKGxpbms9ImxvZ2l0IiksIGRhdGEgPSBDTFVaSCkKCnN1bW1hcnkoc2l6ZV9tb2RlbCkKCnJlcyA8LSBwcmVkaWN0KHNpemVfbW9kZWwsIG5ld2RhdGEgPSBDTFVaSCwgc2UuZml0ID0gVFJVRSwgdHlwZT0icmVzcG9uc2UiKQpDTFVaSCRzaXplX2ZpdCA8LSByZXMkZml0CkNMVVpIJHNpemVfc2UgPC0gcmVzJHNlLmZpdAoKbGVtbWFfbW9kZWwgPSBnYW0oc2NhbGVkX2Ryb3AgfiBsb2coR29sZG1hbl90cmFpbl9sZW1tYXMpLCBmYW1pbHk9YmV0YXIobGluaz0ibG9naXQiKSwgZGF0YSA9IENMVVpIKQoKc3VtbWFyeShsZW1tYV9tb2RlbCkKCnJlcyA8LSBwcmVkaWN0KGxlbW1hX21vZGVsLCBuZXdkYXRhID0gQ0xVWkgsIHNlLmZpdCA9IFRSVUUsIHR5cGU9InJlc3BvbnNlIikKQ0xVWkgkbGVtbWFfZml0IDwtIHJlcyRmaXQKQ0xVWkgkbGVtbWFfc2UgPC0gcmVzJHNlLmZpdAoKZHJvcF9tb2RlbCA9IGdhbShzY2FsZWRfZHJvcCB+IGxvZygtMSp0cmFpbl9sZW1tYV9kaWZmX3JhdyksIGZhbWlseT1iZXRhcihsaW5rPSJsb2dpdCIpLCBkYXRhID0gQ0xVWkgpCgpzdW1tYXJ5KGRyb3BfbW9kZWwpCgpyZXMgPC0gcHJlZGljdChkcm9wX21vZGVsLCBuZXdkYXRhID0gQ0xVWkgsIHNlLmZpdCA9IFRSVUUsIHR5cGU9InJlc3BvbnNlIikKQ0xVWkgkZHJvcF9maXQgPC0gcmVzJGZpdApDTFVaSCRkcm9wX3NlIDwtIHJlcyRzZS5maXQKCmJvdGhfbW9kZWwgPC0gZ2FtKHNjYWxlZF9kcm9wIH4gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSArIGxvZygtMSp0cmFpbl9sZW1tYV9kaWZmX3JhdykgKyBsb2coR29sZG1hbl90cmFpbl9zaXplKSAsIGZhbWlseT1iZXRhcihsaW5rPSJsb2dpdCIpLCBkYXRhID0gQ0xVWkgpCgpzdW1tYXJ5KGJvdGhfbW9kZWwpCgpzaXplX21vZGVsJGNvZWZmaWNpZW50cwpsZW1tYV9tb2RlbCRjb2VmZmljaWVudHMKZHJvcF9tb2RlbCRjb2VmZmljaWVudHMKCkFJQyhzaXplX21vZGVsKQpBSUMobGVtbWFfbW9kZWwpCkFJQyhkcm9wX21vZGVsKQoKCkJJQyhzaXplX21vZGVsKQpCSUMobGVtbWFfbW9kZWwpCkJJQyhkcm9wX21vZGVsKQpgYGAKCmBgYHtyLCBmaWcud2lkdGggPSAxMiwgZmlnLmhlaWdodD00fQpyYmluZChDTFVaSCwgQ0hSVFJNLCBEZWVwU3BpbikgJT4lIAogIGdncGxvdChhZXMoR29sZG1hbl90cmFpbl9zaXplLCAtMSpzY2FsZWRfZHJvcCkpICsgCiAgZ2VvbV9wb2ludChhZXMoY29sb3I9RmFtaWx5KSwgc2l6ZSA9IDMsIGFscGhhID0gMC42KSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygidHVycXVvaXNlIiwgInB1cnBsZSIsICJnb2xkIikpICsgCiAgZmFjZXRfZ3JpZCh+ZmFjdG9yKE1vZGVsLCBsZXZlbHM9YygiQ0hSLVRSTSIsICJEZWVwU3BpbiIsICJDTFVaSCIpKSkgKyAKICBnZW9tX2xpbmUoYWVzKEdvbGRtYW5fdHJhaW5fc2l6ZSwgLTEqc2l6ZV9maXQpKSArIAogIGdlb21fcmliYm9uKGFlcyh4ID0gR29sZG1hbl90cmFpbl9zaXplLCAKICAgICAgICAgICAgICAgICAgeW1pbiA9IC0xKihzaXplX2ZpdCAtIDIgKiBzaXplX3NlKSwKICAgICAgICAgICAgICAgICAgeW1heCA9IC0xKihzaXplX2ZpdCArIDIgKiBzaXplX3NlKSwKICAgICAgICAgICAgICAgICAgKSwgZmlsbCA9ImdyZXkiLAogICAgICAgICAgICAgICAgICBhbHBoYSA9IDAuNCkgKyAKICB0aGVtZV9idygpICsgCiAgeGxhYigiR29sZG1hbiBUcmFpbmluZyBTaXplIikgKyAKICB5bGFiKCJTY2FsZWQgQWNjdXJhY3kgRHJvcCIpICsgCiAgZ2d0aXRsZSgiVHJhaW5pbmcgU2l6ZSB2cy4gQWNjdXJhY3kgRHJvcCIpICsgCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdD0wLjUsIHNpemU9MTYpKSAKCmdnc2F2ZSgiZmlndXJlcy90cmFpbl9zaXplX2Ryb3AucGRmIiwgZGV2aWNlPSJwZGYiLCB3aWR0aCA9IDEyLCBoZWlnaHQ9NCkKYGBgCgpgYGB7ciwgZmlnLndpZHRoID0gMTIsIGZpZy5oZWlnaHQ9NH0KcmJpbmQoQ0xVWkgsIENIUlRSTSwgRGVlcFNwaW4pICU+JSAKICBnZ3Bsb3QoYWVzKEdvbGRtYW5fdHJhaW5fbGVtbWFzLCAtMSpzY2FsZWRfZHJvcCkpICsgCiAgZ2VvbV9wb2ludChhZXMoY29sb3I9RmFtaWx5KSwgc2l6ZSA9IDMsIGFscGhhID0gMC42KSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygidHVycXVvaXNlIiwgInB1cnBsZSIsICJnb2xkIikpICsgCiAgZmFjZXRfZ3JpZCh+ZmFjdG9yKE1vZGVsLCBsZXZlbHM9YygiQ0hSLVRSTSIsICJEZWVwU3BpbiIsICJDTFVaSCIpKSkgKyAKICBnZW9tX2xpbmUoYWVzKEdvbGRtYW5fdHJhaW5fbGVtbWFzLCAtMSpsZW1tYV9maXQpKSArIAogIGdlb21fcmliYm9uKGFlcyh4ID0gR29sZG1hbl90cmFpbl9sZW1tYXMsIAogICAgICAgICAgICAgICAgICB5bWluID0gLTEqKGxlbW1hX2ZpdCAtIDIgKiBsZW1tYV9zZSksCiAgICAgICAgICAgICAgICAgIHltYXggPSAtMSoobGVtbWFfZml0ICsgMiAqIGxlbW1hX3NlKSwKICAgICAgICAgICAgICAgICAgKSwgZmlsbCA9ImdyZXkiLAogICAgICAgICAgICAgICAgICBhbHBoYSA9IDAuNCkgKyAKICB0aGVtZV9idygpICsgCiAgeGxhYigiR29sZG1hbiBUcmFpbmluZyBMZW1tYXMiKSArIAogIHlsYWIoIlNjYWxlZCBBY2N1cmFjeSBEcm9wIikgKwogIGdndGl0bGUoIlRyYWluaW5nIExlbW1hcyB2cy4gQWNjdXJhY3kgRHJvcCIpICsgCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdD0wLjUsIHNpemU9MTYpKSAKCmdnc2F2ZSgiZmlndXJlcy90cmFpbl9sZW1tYXNfZHJvcC5wZGYiLCBkZXZpY2U9InBkZiIsIHdpZHRoID0gMTIsIGhlaWdodD00KQpgYGAKCmBgYHtyLCBmaWcud2lkdGggPSAxMiwgZmlnLmhlaWdodD00fQoKcmJpbmQoQ0xVWkgsIENIUlRSTSwgRGVlcFNwaW4pICU+JSAKICBnZ3Bsb3QoYWVzKC0xKnRyYWluX2xlbW1hX2RpZmZfcmF3LCAtMSpzY2FsZWRfZHJvcCkpICsgCiAgZ2VvbV9wb2ludChhZXMoY29sb3I9RmFtaWx5KSwgc2l6ZSA9IDMsIGFscGhhID0gMC42KSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygidHVycXVvaXNlIiwgInB1cnBsZSIsICJnb2xkIikpICsgCiAgZmFjZXRfZ3JpZCh+ZmFjdG9yKE1vZGVsLCBsZXZlbHM9YygiQ0hSLVRSTSIsICJEZWVwU3BpbiIsICJDTFVaSCIpKSkgKyAKICBnZW9tX2xpbmUoYWVzKC0xKnRyYWluX2xlbW1hX2RpZmZfcmF3LCAtMSpkcm9wX2ZpdCkpICsgCiAgZ2VvbV9yaWJib24oYWVzKHggPSAtMSp0cmFpbl9sZW1tYV9kaWZmX3JhdywgCiAgICAgICAgICAgICAgICAgIHltaW4gPSAtMSooZHJvcF9maXQgLSAyICogZHJvcF9zZSksCiAgICAgICAgICAgICAgICAgIHltYXggPSAtMSooZHJvcF9maXQgKyAyICogZHJvcF9zZSksCiAgICAgICAgICAgICAgICAgICksIGZpbGwgPSJncmV5IiwKICAgICAgICAgICAgICAgICAgYWxwaGEgPSAwLjQpICsgCiAgdGhlbWVfYncoKSArIAogIHhsYWIoIkRyb3AgaW4gTGVtbWFzIGZyb20gU0lHTU9SUEhPTiB0byBHb2xkbWFuIGV0IGFsLiIpICsgCiAgeWxhYigiU2NhbGVkIEFjY3VyYWN5IERyb3AiKSArCiAgZ2d0aXRsZSgiVHJhaW5pbmcgTGVtbWEgRHJvcCB2cy4gQWNjdXJhY3kgRHJvcCIpICsgCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdD0wLjUsIHNpemU9MTYpKSAKCmdnc2F2ZSgiZmlndXJlcy9sZW1tYV9kcm9wX2Ryb3AucGRmIiwgZGV2aWNlPSJwZGYiLCB3aWR0aCA9IDEyLCBoZWlnaHQ9NCkKYGBgCgojIE90aGVyIFJlZ3Jlc3Npb25zCgojIyBMaW5lYXIgUmVncmVzc2lvbjogUmF3IEFjY3VyYWN5CgpgYGB7cn0Kc2l6ZV9tb2RlbCA9IGxtKGxvZyhHb2xkbWFuX3NjYWxlZF9hY2MpIH4gbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksICBkYXRhID0gQ0hSVFJNKQpzdW1tYXJ5KHNpemVfbW9kZWwpCnJlcyA8LSBwcmVkaWN0KHNpemVfbW9kZWwsIG5ld2RhdGEgPSBDSFJUUk0sIHNlLmZpdCA9IFRSVUUsIHR5cGUgPSAicmVzcG9uc2UiKQpDSFJUUk0kc2l6ZV9maXQgPC0gcmVzJGZpdApDSFJUUk0kc2l6ZV9zZSA8LSByZXMkc2UuZml0CgoKbGVtbWFfbW9kZWwgPSBsbShsb2coR29sZG1hbl9zY2FsZWRfYWNjKSB+IGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcyksIGRhdGEgPSBDSFJUUk0pCnJlcyA8LSBwcmVkaWN0KGxlbW1hX21vZGVsLCBuZXdkYXRhID0gQ0hSVFJNLCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKQ0hSVFJNJGxlbW1hX2ZpdCA8LSByZXMkZml0CkNIUlRSTSRsZW1tYV9zZSA8LSByZXMkc2UuZml0CnN1bW1hcnkobGVtbWFfbW9kZWwpCgpib3RoX21vZGVsIDwtIGxtKGxvZyhHb2xkbWFuX3NjYWxlZF9hY2MpIH4gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSArIGxvZyhHb2xkbWFuX3RyYWluX3NpemUpLCBkYXRhID0gQ0hSVFJNKQpzdW1tYXJ5KGJvdGhfbW9kZWwpCkFJQyhzaXplX21vZGVsKQpBSUMobGVtbWFfbW9kZWwpCkFJQyhib3RoX21vZGVsKQphbm92YShib3RoX21vZGVsLCBsZW1tYV9tb2RlbCkKYGBgCgpgYGB7cn0Kc2l6ZV9tb2RlbCA9IGxtKGxvZyhHb2xkbWFuX3NjYWxlZF9hY2MpIH4gbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksICBkYXRhID0gRGVlcFNwaW4pCnN1bW1hcnkoc2l6ZV9tb2RlbCkKcmVzIDwtIHByZWRpY3Qoc2l6ZV9tb2RlbCwgbmV3ZGF0YSA9IERlZXBTcGluLCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKRGVlcFNwaW4kc2l6ZV9maXQgPC0gcmVzJGZpdApEZWVwU3BpbiRzaXplX3NlIDwtIHJlcyRzZS5maXQKCgpsZW1tYV9tb2RlbCA9IGxtKGxvZyhHb2xkbWFuX3NjYWxlZF9hY2MpIH4gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSwgZGF0YSA9IERlZXBTcGluKQpyZXMgPC0gcHJlZGljdChsZW1tYV9tb2RlbCwgbmV3ZGF0YSA9IERlZXBTcGluLCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKRGVlcFNwaW4kbGVtbWFfZml0IDwtIHJlcyRmaXQKRGVlcFNwaW4kbGVtbWFfc2UgPC0gcmVzJHNlLmZpdApzdW1tYXJ5KGxlbW1hX21vZGVsKQoKYm90aF9tb2RlbCA8LSBsbShsb2coR29sZG1hbl9zY2FsZWRfYWNjKSB+IGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcykgKyBsb2coR29sZG1hbl90cmFpbl9zaXplKSwgZGF0YSA9IERlZXBTcGluKQpzdW1tYXJ5KGJvdGhfbW9kZWwpCkFJQyhzaXplX21vZGVsKQpBSUMobGVtbWFfbW9kZWwpCkFJQyhib3RoX21vZGVsKQphbm92YShib3RoX21vZGVsLCBsZW1tYV9tb2RlbCkKYGBgCgpgYGB7cn0Kc2l6ZV9tb2RlbCA9IGxtKGxvZyhHb2xkbWFuX3NjYWxlZF9hY2MpIH4gbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksICBkYXRhID0gQ0xVWkgpCnN1bW1hcnkoc2l6ZV9tb2RlbCkKcmVzIDwtIHByZWRpY3Qoc2l6ZV9tb2RlbCwgbmV3ZGF0YSA9IENMVVpILCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKQ0xVWkgkc2l6ZV9maXQgPC0gcmVzJGZpdApDTFVaSCRzaXplX3NlIDwtIHJlcyRzZS5maXQKCgpsZW1tYV9tb2RlbCA9IGxtKGxvZyhHb2xkbWFuX3NjYWxlZF9hY2MpIH4gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSwgZGF0YSA9IENMVVpIKQpyZXMgPC0gcHJlZGljdChsZW1tYV9tb2RlbCwgbmV3ZGF0YSA9IENMVVpILCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKQ0xVWkgkbGVtbWFfZml0IDwtIHJlcyRmaXQKQ0xVWkgkbGVtbWFfc2UgPC0gcmVzJHNlLmZpdApzdW1tYXJ5KGxlbW1hX21vZGVsKQoKYm90aF9tb2RlbCA8LSBsbShsb2coR29sZG1hbl9zY2FsZWRfYWNjKSB+IGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcykgKyBsb2coR29sZG1hbl90cmFpbl9zaXplKSwgZGF0YSA9IENMVVpIKQpzdW1tYXJ5KGJvdGhfbW9kZWwpCkFJQyhzaXplX21vZGVsKQpBSUMobGVtbWFfbW9kZWwpCkFJQyhib3RoX21vZGVsKQphbm92YShib3RoX21vZGVsLCBsZW1tYV9tb2RlbCkKYGBgCgpgYGB7ciwgZmlnLndpZHRoID0gMTIsIGZpZy5oZWlnaHQ9NH0KcmJpbmQoQ0xVWkgsIENIUlRSTSwgRGVlcFNwaW4pICU+JSAKICBnZ3Bsb3QoYWVzKGxvZyhHb2xkbWFuX3RyYWluX3NpemUpLCBsb2coR29sZG1hbl9zY2FsZWRfYWNjKSkpICsgCiAgZ2VvbV9wb2ludChhZXMoY29sb3I9RmFtaWx5KSwgc2l6ZSA9IDMsIGFscGhhID0gMC42KSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygidHVycXVvaXNlIiwgInB1cnBsZSIsICJnb2xkIikpICsgCiAgZmFjZXRfZ3JpZCh+ZmFjdG9yKE1vZGVsLCBsZXZlbHM9YygiQ0hSLVRSTSIsICJEZWVwU3BpbiIsICJDTFVaSCIpKSkgKyAKICBnZW9tX2xpbmUoYWVzKGxvZyhHb2xkbWFuX3RyYWluX3NpemUpLCBzaXplX2ZpdCkpICsgCiAgZ2VvbV9yaWJib24oYWVzKHggPSBsb2coR29sZG1hbl90cmFpbl9zaXplKSwgCiAgICAgICAgICAgICAgICAgIHltaW4gPSBzaXplX2ZpdCAtIDIgKiBzaXplX3NlLAogICAgICAgICAgICAgICAgICB5bWF4ID0gc2l6ZV9maXQgKyAyICogc2l6ZV9zZSwKICAgICAgICAgICAgICAgICAgKSwgZmlsbCA9ImdyZXkiLAogICAgICAgICAgICAgICAgICBhbHBoYSA9IDAuNCkgKyAKICB0aGVtZV9idygpICsgCiAgeGxhYigiR29sZG1hbiBUcmFpbmluZyBTaXplLCBMb2cgU2NhbGUiKSArIAogIHlsYWIoIkFjY3VyYWN5IikgKyAKICBnZ3RpdGxlKCJUcmFpbmluZyBTaXplIHZzLiBBY2N1cmFjeSwgR29sZG1hbiBldCBhbC4gRGF0YSIpICsgCiAgdGhlbWUocGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChoanVzdD0wLjUsIHNpemU9MTYpKSAKCmdnc2F2ZSgiZmlndXJlcy90cmFpbl9zaXplX2xtLnBkZiIsIGRldmljZT0icGRmIiwgd2lkdGggPSAxMiwgaGVpZ2h0PTQpCmBgYAoKYGBge3IsIGZpZy53aWR0aCA9IDEyLCBmaWcuaGVpZ2h0PTR9CnJiaW5kKENMVVpILCBDSFJUUk0sIERlZXBTcGluKSAlPiUgCiAgZ2dwbG90KGFlcyhsb2coR29sZG1hbl90cmFpbl9sZW1tYXMpLCBsb2coR29sZG1hbl9zY2FsZWRfYWNjKSkpICsgCiAgZ2VvbV9wb2ludChhZXMoY29sb3I9RmFtaWx5KSwgc2l6ZSA9IDMsIGFscGhhID0gMC42KSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygidHVycXVvaXNlIiwgInB1cnBsZSIsICJnb2xkIikpICsgCiAgZmFjZXRfZ3JpZCh+ZmFjdG9yKE1vZGVsLCBsZXZlbHM9YygiQ0hSLVRSTSIsICJEZWVwU3BpbiIsICJDTFVaSCIpKSkgKyAKICBnZW9tX2xpbmUoYWVzKGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcyksIGxlbW1hX2ZpdCkpICsgCiAgZ2VvbV9yaWJib24oYWVzKHggPSBsb2coR29sZG1hbl90cmFpbl9sZW1tYXMpLCAKICAgICAgICAgICAgICAgICAgeW1pbiA9IGxlbW1hX2ZpdCAtIDIgKiBsZW1tYV9zZSwKICAgICAgICAgICAgICAgICAgeW1heCA9IGxlbW1hX2ZpdCArIDIgKiBsZW1tYV9zZSwKICAgICAgICAgICAgICAgICAgKSwgZmlsbCA9ImdyZXkiLAogICAgICAgICAgICAgICAgICBhbHBoYSA9IDAuNCkgKyAKICB0aGVtZV9idygpICsgCiAgeGxhYigiR29sZG1hbiBUcmFpbmluZyBMZW1tYXMsIExvZyBTY2FsZSIpICsgCiAgeWxhYigiQWNjdXJhY3kiKSArIAogIGdndGl0bGUoIlRyYWluaW5nIExlbW1hcyB2cy4gQWNjdXJhY3ksIEdvbGRtYW4gZXQgYWwuIERhdGEiKSArIAogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3Q9MC41LCBzaXplPTE2KSkgCgpnZ3NhdmUoImZpZ3VyZXMvdHJhaW5fbGVtbWFzX2xtLnBkZiIsIGRldmljZT0icGRmIiwgd2lkdGggPSAxMiwgaGVpZ2h0PTQpCmBgYAoKIyMgTG9naXN0aWMgUmVncmVzc2lvbjogUmF3IEFjY3VyYWN5CgpgYGB7cn0Kc2l6ZV9tb2RlbCA9IGdsbShHb2xkbWFuX3NjYWxlZF9hY2MgfiBsb2coR29sZG1hbl90cmFpbl9zaXplKSwgIGRhdGEgPSBDSFJUUk0sIGZhbWlseT1iaW5vbWlhbCkKc3VtbWFyeShzaXplX21vZGVsKQpyZXMgPC0gcHJlZGljdChzaXplX21vZGVsLCBuZXdkYXRhID0gQ0hSVFJNLCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKQ0hSVFJNJHNpemVfZml0IDwtIHJlcyRmaXQKQ0hSVFJNJHNpemVfc2UgPC0gcmVzJHNlLmZpdAoKCmxlbW1hX21vZGVsID0gZ2xtKEdvbGRtYW5fc2NhbGVkX2FjYyB+IGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcyksIGRhdGEgPSBDSFJUUk0sIGZhbWlseT1iaW5vbWlhbCkKcmVzIDwtIHByZWRpY3QobGVtbWFfbW9kZWwsIG5ld2RhdGEgPSBDSFJUUk0sIHNlLmZpdCA9IFRSVUUsIHR5cGUgPSAicmVzcG9uc2UiKQpDSFJUUk0kbGVtbWFfZml0IDwtIHJlcyRmaXQKQ0hSVFJNJGxlbW1hX3NlIDwtIHJlcyRzZS5maXQKc3VtbWFyeShsZW1tYV9tb2RlbCkKCmJvdGhfbW9kZWwgPC0gZ2xtKEdvbGRtYW5fc2NhbGVkX2FjYyB+IGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcykgKyBsb2coR29sZG1hbl90cmFpbl9zaXplKSwgZGF0YSA9IENIUlRSTSwgZmFtaWx5PWJpbm9taWFsKQpzdW1tYXJ5KGJvdGhfbW9kZWwpCkFJQyhzaXplX21vZGVsKQpBSUMobGVtbWFfbW9kZWwpCkFJQyhib3RoX21vZGVsKQphbm92YShib3RoX21vZGVsLCBsZW1tYV9tb2RlbCkKYGBgCgpgYGB7cn0Kc2l6ZV9tb2RlbCA9IGdsbShHb2xkbWFuX3NjYWxlZF9hY2MgfiBsb2coR29sZG1hbl90cmFpbl9zaXplKSwgIGRhdGEgPSBEZWVwU3BpbiwgZmFtaWx5PWJpbm9taWFsKQpzdW1tYXJ5KHNpemVfbW9kZWwpCnJlcyA8LSBwcmVkaWN0KHNpemVfbW9kZWwsIG5ld2RhdGEgPSBEZWVwU3Bpbiwgc2UuZml0ID0gVFJVRSwgdHlwZSA9ICJyZXNwb25zZSIpCkRlZXBTcGluJHNpemVfZml0IDwtIHJlcyRmaXQKRGVlcFNwaW4kc2l6ZV9zZSA8LSByZXMkc2UuZml0CgoKbGVtbWFfbW9kZWwgPSBnbG0oR29sZG1hbl9zY2FsZWRfYWNjIH4gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSwgZGF0YSA9IERlZXBTcGluLCBmYW1pbHk9Ymlub21pYWwpCnJlcyA8LSBwcmVkaWN0KGxlbW1hX21vZGVsLCBuZXdkYXRhID0gQ0hSVFJNLCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKRGVlcFNwaW4kbGVtbWFfZml0IDwtIHJlcyRmaXQKRGVlcFNwaW4kbGVtbWFfc2UgPC0gcmVzJHNlLmZpdApzdW1tYXJ5KGxlbW1hX21vZGVsKQoKYm90aF9tb2RlbCA8LSBnbG0oR29sZG1hbl9zY2FsZWRfYWNjIH4gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSArIGxvZyhHb2xkbWFuX3RyYWluX3NpemUpLCBkYXRhID0gRGVlcFNwaW4sIGZhbWlseT1iaW5vbWlhbCkKc3VtbWFyeShib3RoX21vZGVsKQpBSUMoc2l6ZV9tb2RlbCkKQUlDKGxlbW1hX21vZGVsKQpBSUMoYm90aF9tb2RlbCkKYW5vdmEoYm90aF9tb2RlbCwgbGVtbWFfbW9kZWwpCmBgYAoKYGBge3J9CnNpemVfbW9kZWwgPSBnbG0oR29sZG1hbl9zY2FsZWRfYWNjIH4gbG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksICBkYXRhID0gQ0xVWkgsIGZhbWlseT1iaW5vbWlhbCkKc3VtbWFyeShzaXplX21vZGVsKQpyZXMgPC0gcHJlZGljdChzaXplX21vZGVsLCBuZXdkYXRhID0gQ0xVWkgsIHNlLmZpdCA9IFRSVUUsIHR5cGUgPSAicmVzcG9uc2UiKQpDTFVaSCRzaXplX2ZpdCA8LSByZXMkZml0CkNMVVpIJHNpemVfc2UgPC0gcmVzJHNlLmZpdAoKCmxlbW1hX21vZGVsID0gZ2xtKEdvbGRtYW5fc2NhbGVkX2FjYyB+IGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcyksIGRhdGEgPSBDTFVaSCwgZmFtaWx5PWJpbm9taWFsKQpyZXMgPC0gcHJlZGljdChsZW1tYV9tb2RlbCwgbmV3ZGF0YSA9IENMVVpILCBzZS5maXQgPSBUUlVFLCB0eXBlID0gInJlc3BvbnNlIikKQ0xVWkgkbGVtbWFfZml0IDwtIHJlcyRmaXQKQ0xVWkgkbGVtbWFfc2UgPC0gcmVzJHNlLmZpdApzdW1tYXJ5KGxlbW1hX21vZGVsKQoKYm90aF9tb2RlbCA8LSBnbG0oR29sZG1hbl9zY2FsZWRfYWNjIH4gbG9nKEdvbGRtYW5fdHJhaW5fbGVtbWFzKSArIGxvZyhHb2xkbWFuX3RyYWluX3NpemUpLCBkYXRhID0gQ0xVWkgsIGZhbWlseT1iaW5vbWlhbCkKc3VtbWFyeShib3RoX21vZGVsKQpBSUMoc2l6ZV9tb2RlbCkKQUlDKGxlbW1hX21vZGVsKQpBSUMoYm90aF9tb2RlbCkKYW5vdmEoYm90aF9tb2RlbCwgbGVtbWFfbW9kZWwpCmBgYAoKYGBge3IsIGZpZy53aWR0aCA9IDEyLCBmaWcuaGVpZ2h0PTR9CnJiaW5kKENMVVpILCBDSFJUUk0sIERlZXBTcGluKSAlPiUgCiAgZ2dwbG90KGFlcyhsb2coR29sZG1hbl90cmFpbl9zaXplKSwgR29sZG1hbl9zY2FsZWRfYWNjKSkgKyAKICBnZW9tX3BvaW50KGFlcyhjb2xvcj1GYW1pbHkpLCBzaXplID0gMywgYWxwaGEgPSAwLjYpICsgCiAgc2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcz1jKCJ0dXJxdW9pc2UiLCAicHVycGxlIiwgImdvbGQiKSkgKyAKICBmYWNldF9ncmlkKH5mYWN0b3IoTW9kZWwsIGxldmVscz1jKCJDSFItVFJNIiwgIkRlZXBTcGluIiwgIkNMVVpIIikpKSArIAogIGdlb21fbGluZShhZXMobG9nKEdvbGRtYW5fdHJhaW5fc2l6ZSksIHNpemVfZml0KSkgKyAKICBnZW9tX3JpYmJvbihhZXMoeCA9IGxvZyhHb2xkbWFuX3RyYWluX3NpemUpLCAKICAgICAgICAgICAgICAgICAgeW1pbiA9IHNpemVfZml0IC0gMiAqIHNpemVfc2UsCiAgICAgICAgICAgICAgICAgIHltYXggPSBzaXplX2ZpdCArIDIgKiBzaXplX3NlLAogICAgICAgICAgICAgICAgICApLCBmaWxsID0iZ3JleSIsCiAgICAgICAgICAgICAgICAgIGFscGhhID0gMC40KSArIAogIHRoZW1lX2J3KCkgKyAKICB4bGFiKCJHb2xkbWFuIFRyYWluaW5nIFNpemUsIExvZyBTY2FsZSIpICsgCiAgeWxhYigiQWNjdXJhY3kiKSArIAogIGdndGl0bGUoIlRyYWluaW5nIFNpemUgdnMuIEFjY3VyYWN5LCBHb2xkbWFuIGV0IGFsLiBEYXRhIikgKyAKICB0aGVtZShwbG90LnRpdGxlID0gZWxlbWVudF90ZXh0KGhqdXN0PTAuNSwgc2l6ZT0xNikpIAoKZ2dzYXZlKCJmaWd1cmVzL3RyYWluX3NpemVfbG9naXN0aWMucGRmIiwgZGV2aWNlPSJwZGYiLCB3aWR0aCA9IDEyLCBoZWlnaHQ9NCkKYGBgCgpgYGB7ciwgZmlnLndpZHRoID0gMTIsIGZpZy5oZWlnaHQ9NH0KcmJpbmQoQ0xVWkgsIENIUlRSTSwgRGVlcFNwaW4pICU+JSAKICBnZ3Bsb3QoYWVzKGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcyksIEdvbGRtYW5fc2NhbGVkX2FjYykpICsgCiAgZ2VvbV9wb2ludChhZXMoY29sb3I9RmFtaWx5KSwgc2l6ZSA9IDMsIGFscGhhID0gMC42KSArIAogIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygidHVycXVvaXNlIiwgInB1cnBsZSIsICJnb2xkIikpICsgCiAgZmFjZXRfZ3JpZCh+ZmFjdG9yKE1vZGVsLCBsZXZlbHM9YygiQ0hSLVRSTSIsICJEZWVwU3BpbiIsICJDTFVaSCIpKSkgKyAKICBnZW9tX2xpbmUoYWVzKGxvZyhHb2xkbWFuX3RyYWluX2xlbW1hcyksIGxlbW1hX2ZpdCkpICsgCiAgZ2VvbV9yaWJib24oYWVzKHggPSBsb2coR29sZG1hbl90cmFpbl9sZW1tYXMpLCAKICAgICAgICAgICAgICAgICAgeW1pbiA9IGxlbW1hX2ZpdCAtIDIgKiBsZW1tYV9zZSwKICAgICAgICAgICAgICAgICAgeW1heCA9IGxlbW1hX2ZpdCArIDIgKiBsZW1tYV9zZSwKICAgICAgICAgICAgICAgICAgKSwgZmlsbCA9ImdyZXkiLAogICAgICAgICAgICAgICAgICBhbHBoYSA9IDAuNCkgKyAKICB0aGVtZV9idygpICsgCiAgeGxhYigiR29sZG1hbiBUcmFpbmluZyBMZW1tYXMsIExvZyBTY2FsZSIpICsgCiAgeWxhYigiQWNjdXJhY3kiKSArIAogIGdndGl0bGUoIlRyYWluaW5nIExlbW1hcyB2cy4gQWNjdXJhY3ksIEdvbGRtYW4gZXQgYWwuIERhdGEiKSArIAogIHRoZW1lKHBsb3QudGl0bGUgPSBlbGVtZW50X3RleHQoaGp1c3Q9MC41LCBzaXplPTE2KSkgCgpnZ3NhdmUoImZpZ3VyZXMvdHJhaW5fbGVtbWFzX2xvZ2lzdGljLnBkZiIsIGRldmljZT0icGRmIiwgd2lkdGggPSAxMiwgaGVpZ2h0PTQpCmBgYAo=